Bug: Collations (#21)

Author: dokshina

crud/common/collations.lua

@@ -0,0 +1,48 @@
+local json = require('json')
+
+local COLLATION_NAME_FN = 2
+
+local collations = {}
+
+collations.NONE = 'none'
+collations.BINARY = 'binary'
+collations.UNICODE = 'unicode'
+collations.UNICODE_CI = 'unicode_ci'
+
+function collations.get(index_part)
+    if index_part.collation ~= nil then
+        return index_part.collation
+    end
+
+    if index_part.collation_id == nil then
+        return collations.NONE
+    end
+
+    local collation_tuple = box.space._collation:get(index_part.collation_id)
+    assert(collation_tuple ~= nil, "Unknown collation_id: " .. json.encode(index_part.collation_id))
+
+    local collation = collation_tuple[COLLATION_NAME_FN]
+    return collation
+end
+
+function collations.is_default(collation)
+    if collation == nil then
+        return true
+    end
+
+    if collation == collations.NONE or collation == collations.BINARY then
+        return true
+    end
+
+    return false
+end
+
+function collations.is_unicode(collation)
+    if collation == nil then
+        return false
+    end
+
+    return string.startswith(collation, 'unicode_')
+end
+
+return collations

crud/select/comparators.lua

@@ -1,5 +1,6 @@
 local errors = require('errors')
 
+local collations = require('crud.common.collations')
 local select_conditions = require('crud.select.conditions')
 local operators = select_conditions.operators
 
@@ -131,17 +132,18 @@ local function gen_array_cmp_func(target, key_parts)
     local eq_funcs = {}
 
     for _, part in ipairs(key_parts) do
-        if part.collation == nil then
+        local collation = collations.get(part)
+        if collations.is_default(collation) then
             table.insert(lt_funcs, lt)
             table.insert(eq_funcs, eq)
-        elseif part.collation == 'unicode' then
+        elseif collation == collations.UNICODE then
             table.insert(lt_funcs, lt_unicode)
             table.insert(eq_funcs, eq_unicode)
-        elseif part.collation == 'unicode_ci' then
+        elseif collation == collations.UNICODE_CI then
             table.insert(lt_funcs, lt_unicode_ci)
             table.insert(eq_funcs, eq_unicode_ci)
         else
-            return nil, GenFuncError:new('Unsupported tarantool collation %q', part.collation)
+            return nil, GenFuncError:new('Unsupported Tarantool collation %q', collation)
         end
     end
 
@@ -175,7 +177,7 @@ local array_cmp_funcs_by_operators = {
 --]=]
 function comparators.get_cmp_operator(tarantool_iter)
     local cmp_operator = cmp_operators_by_tarantool_iter[tarantool_iter]
-    assert(cmp_operator ~= nil, 'Unknown Tarantool iterator %q', tarantool_iter)
+    assert(cmp_operator ~= nil, 'Unsupported Tarantool iterator %q', tarantool_iter)
 
     return cmp_operator
 end

crud/select/filters.lua

@@ -1,3 +1,4 @@
+local collations = require('crud.common.collations')
 local select_conditions = require('crud.select.conditions')
 
 local function format_value(value)
@@ -73,6 +74,30 @@ local function format_comp_with_value(fieldno, func_name, value)
     )
 end
 
+local function add_strict_postfix(func_name, value_opts)
+    if value_opts.is_nullable == false then
+        return string.format('%s_strict', func_name)
+    end
+
+    return func_name
+end
+
+local function add_collation_postfix(func_name, value_opts)
+    if collations.is_default(value_opts.collation) then
+        return func_name
+    end
+
+    if value_opts.collation == collations.UNICODE then
+        return string.format('%s_unicode', func_name)
+    end
+
+    if value_opts.collation == collations.UNICODE_CI then
+        return string.format('%s_unicode_ci', func_name)
+    end
+
+    error('Unsupported collation: ' .. tostring(value_opts.collation))
+end
+
 local function format_eq(cond)
     local cond_strings = {}
     local values_opts = cond.values_opts or {}
@@ -82,22 +107,11 @@ local function format_eq(cond)
         local value = cond.values[j]
         local value_opts = values_opts[j] or {}
 
-        local postfix = ''
-        if value_opts.is_nullable == false then
-            postfix = '_strict'
-        end
+        local func_name = 'eq'
+        func_name = add_collation_postfix(func_name, value_opts)
 
-        local func_name
-        if value_opts.collation ~= nil then
-            if value_opts.collation == 'unicode' then
-                func_name = 'eq_unicode' .. postfix
-            elseif value_opts.collation == 'unicode_ci' then
-                func_name = 'eq_unicode_ci' .. postfix
-            else
-                error('unknown collation: ' .. tostring(value_opts.collation))
-            end
-        else
-            func_name = 'eq'
+        if collations.is_unicode(value_opts.collation) then
+            func_name = add_strict_postfix(func_name, value_opts)
         end
 
         table.insert(cond_strings, format_comp_with_value(fieldno, func_name, value))
@@ -116,25 +130,9 @@ local function format_lt(cond)
         local value_type = cond.types[j]
         local value_opts = values_opts[j] or {}
 
-        local postfix = ''
-        if value_opts.is_nullable == false then
-            postfix = '_strict'
-        end
-
-        local func_name
-        if value_opts.collation ~= nil then
-            if value_opts.collation == 'unicode' then
-                func_name = 'lt_unicode' .. postfix
-            elseif value_opts.collation == 'unicode_ci' then
-                func_name = 'lt_unicode_ci' .. postfix
-            else
-                error('unknown collation: ' .. tostring(value_opts.collation))
-            end
-        elseif value_type == 'boolean' then
-            func_name = 'lt_boolean' .. postfix
-        else
-            func_name = 'lt' .. postfix
-        end
+        local func_name = value_type ~= 'boolean' and 'lt' or 'lt_boolean'
+        func_name = add_collation_postfix(func_name, value_opts)
+        func_name = add_strict_postfix(func_name, value_opts)
 
         table.insert(cond_strings, format_comp_with_value(fieldno, func_name, value))
     end
@@ -405,7 +403,7 @@ local library = {
     -- LT
     -- nullable
     lt = lt_nullable,
-    lt_unicode_ = lt_unicode_nullable,
+    lt_unicode = lt_unicode_nullable,
     lt_unicode_ci = lt_unicode_ci_nullable,
     lt_boolean = lt_boolean_nullable,
     -- strict

crud/select/plan.lua

@@ -1,6 +1,7 @@
 local errors = require('errors')
 local json = require('json')
 
+local collations = require('crud.common.collations')
 local select_conditions = require('crud.select.conditions')
 
 local select_plan = {}
@@ -180,7 +181,7 @@ local function get_values_opts(index, fieldnos)
             assert(index_part ~= nil)
 
             is_nullable = index_part.is_nullable
-            collation = index_part.collation
+            collation = collations.get(index_part)
         end
 
         table.insert(values_opts, {

test/entrypoint/srv_select.lua

@@ -41,7 +41,7 @@ package.preload['customers-storage'] = function()
             customers_space:create_index('full_name', {
                 parts = {
                     { field = 'name', collation = 'unicode_ci' },
-                    { field = 'last_name', is_nullable = true },
+                    { field = 'last_name', collation = 'unicode_ci' },
                 },
                 unique = false,
                 if_not_exists = true,

test/integration/select_test.lua

@@ -751,3 +751,57 @@ add('test_select_by_full_sharding_key', function(g)
     t.assert_equals(err, nil)
     t.assert_equals(objects, get_by_ids(customers, {3}))
 end)
+
+add('test_select_with_collations', function(g)
+    local customers = insert_customers(g, {
+        {
+            id = 1, name = "Elizabeth", last_name = "Jackson",
+            age = 12, city = "Oxford",
+        }, {
+            id = 2, name = "Mary", last_name = "Brown",
+            age = 46, city = "oxford",
+        }, {
+            id = 3, name = "elizabeth", last_name = "brown",
+            age = 46, city = "Oxford",
+        }, {
+            id = 4, name = "Jack", last_name = "Sparrow",
+            age = 35, city = "oxford",
+        }, {
+            id = 5, name = "William", last_name = "Terner",
+            age = 25, city = "Oxford",
+        }, {
+            id = 6, name = "elizabeth", last_name = "Brown",
+            age = 33, city = "Los Angeles",
+        },
+    })
+
+    table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end)
+
+    -- full name index - unicode ci collation (case-insensitive)
+    local conditions = {{'==', 'name', "Elizabeth"}}
+    local objects, err = g.cluster.main_server.net_box:eval([[
+        local crud = require('crud')
+
+        local conditions = ...
+
+        local objects, err = crud.select('customers', conditions)
+        return objects, err
+    ]], {conditions})
+
+    t.assert_equals(err, nil)
+    t.assert_equals(objects, get_by_ids(customers, {3, 6, 1}))
+
+    -- city - no collation (case-sensitive)
+    local conditions = {{'==', 'city', "oxford"}}
+    local objects, err = g.cluster.main_server.net_box:eval([[
+        local crud = require('crud')
+
+        local conditions = ...
+
+        local objects, err = crud.select('customers', conditions)
+        return objects, err
+    ]], {conditions})
+
+    t.assert_equals(err, nil)
+    t.assert_equals(objects, get_by_ids(customers, {2, 4}))
+end)

test/unit/select_filters_test.lua

@@ -311,7 +311,7 @@ g.test_unicode_collation = function()
             types = {'string', 'string', 'string', 'number'},
             early_exit_is_possible = false,
             values_opts = {
-                {collation='unicode_ci'},
+                {collation='unicode'},
                 {collation='unicode_ci'},
                 {collation='unicode_ci'},
             }
@@ -332,7 +332,7 @@ return true, false]]
     local expected_library_code = [[local M = {}
 
 function M.eq_1(field_1, field_2, field_3, field_4)
-    return (eq_unicode_ci(field_1, "A") and eq_unicode_ci(field_2, "Á") and eq_unicode_ci(field_3, "Ä") and eq(field_4, 6))
+    return (eq_unicode(field_1, "A") and eq_unicode_ci(field_2, "Á") and eq_unicode_ci(field_3, "Ä") and eq(field_4, 6))
 end
 
 return M]]
@@ -343,8 +343,54 @@ return M]]
 
     local func = select_filters.compile(filter)
     t.assert_equals(func({'A', 'Á', 'Ä', 6}), true)
-    t.assert_equals(func({'a', 'á', 'ä', 6}), true)
-    t.assert_equals(func({'a', 'V', 'ä', 6}), false)
+    t.assert_equals(func({'A', 'á', 'ä', 6}), true)
+    t.assert_equals(func({'a', 'Á', 'Ä', 6}), false)
+    t.assert_equals(func({'A', 'V', 'ä', 6}), false)
+end
+
+g.test_binary_and_none_collation = function()
+    local filter_conditions = {
+        {
+            fieldnos = {1, 2, 3},
+            operator = select_conditions.operators.EQ,
+            values = {'A', 'B', 'C'},
+            types = {'string', 'string', 'string'},
+            early_exit_is_possible = false,
+            values_opts = {
+                {collation='none'},
+                {collation='binary'},
+                {collation=nil},
+            }
+        },
+    }
+
+    local expected_code = [[local tuple = ...
+
+local field_1 = tuple[1]
+local field_2 = tuple[2]
+local field_3 = tuple[3]
+
+if not eq_1(field_1, field_2, field_3) then return false, false end
+
+return true, false]]
+
+    local expected_library_code = [[local M = {}
+
+function M.eq_1(field_1, field_2, field_3)
+    return (eq(field_1, "A") and eq(field_2, "B") and eq(field_3, "C"))
+end
+
+return M]]
+
+    local filter = select_filters.gen_code(filter_conditions)
+    t.assert_equals(filter.code, expected_code)
+    t.assert_equals(filter.library_code, expected_library_code)
+
+    local func = select_filters.compile(filter)
+    t.assert_equals(func({'A', 'B', 'C'}), true)
+    t.assert_equals(func({'a', 'B', 'C'}), false)
+    t.assert_equals(func({'A', 'b', 'C'}), false)
+    t.assert_equals(func({'A', 'B', 'c'}), false)
 end
 
 g.test_null_as_last_value_eq = function()

test/unit/select_plan_test.lua

@@ -1,5 +1,6 @@
 local select_plan = require('crud.select.plan')
 
+local collations = require('crud.common.collations')
 local select_conditions = require('crud.select.conditions')
 local cond_funcs = select_conditions.funcs
 
@@ -228,12 +229,12 @@ g.test_filter_conditions = function()
     -- - name part opts
     local name_opts = full_name_values_opts[1]
     t.assert_equals(name_opts.is_nullable, false)
-    t.assert_equals(name_opts.collation, 'unicode_ci')
+    t.assert_equals(name_opts.collation, collations.UNICODE_CI)
 
     -- - last_name part opts
     local last_name_opts = full_name_values_opts[2]
     t.assert_equals(last_name_opts.is_nullable, true)
-    t.assert_equals(last_name_opts.collation, nil)
+    t.assert_equals(last_name_opts.collation, collations.NONE)
 
     -- has_a_car filter
     local has_a_car_filter_condition = plan.filter_conditions[3]