RCORE-2170 String compression tests (#7812)

Author: nicola-cab

evergreen/config.yml

@@ -1863,19 +1863,18 @@ buildvariants:
   - name: finalize_coverage_data
 
 - name: macos-array-compression
-  display_name: "MacOS 11 arm64 (Compress Arrays)"
-  run_on: macos-1100-arm64
+  display_name: "MacOS 14 arm64 (Compress Arrays)"
+  run_on: macos-14-arm64
   expansions:
-    cmake_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/cmake-3.26.3-macos-universal.tar.gz"
-    cmake_bindir: "./cmake_binaries/CMake.app/Contents/bin"
+    cmake_bindir: "/opt/homebrew/bin"
     cmake_toolchain_file: "./tools/cmake/xcode.toolchain.cmake"
+    cmake_build_tool_options: "-sdk macosx"
     cmake_generator: Xcode
     max_jobs: $(sysctl -n hw.logicalcpu)
-    xcode_developer_dir: /Applications/Xcode13.1.app/Contents/Developer
+    xcode_developer_dir: /Applications/Xcode15.2.app/Contents/Developer
     extra_flags: -DCMAKE_SYSTEM_NAME=Darwin -DCMAKE_OSX_ARCHITECTURES=arm64
     compress: On
     cmake_build_type: Debug
-    coveralls_flag_name: "macos-arm64"
   tasks:
   - name: compile_test
 

src/realm/string_interner.cpp

@@ -223,7 +223,7 @@ static std::vector<uint32_t> hash_to_id(Array& node, uint32_t hash, uint8_t hash
     if (!node.has_refs()) {
         // it's a leaf - default is a list, search starts from index 0.
         HashMapIter it(node, hash, hash_size);
-        if (node.size() > hash_node_min_size) {
+        if (node.size() >= hash_node_min_size) {
             // it is a hash table, so use hash to select index to start searching
             // table size must be power of two!
             size_t index = hash & (node.size() - 1);
@@ -590,6 +590,7 @@ CompressedStringView& StringInterner::get_compressed(StringID id)
     auto index = id - 1; // 0 represents null
     auto hi = index >> 8;
     auto lo = index & 0xFFUL;
+
     DataLeaf& leaf = m_compressed_leafs[hi];
     load_leaf_if_needed(leaf);
     REALM_ASSERT_DEBUG(lo < leaf.m_compressed.size());
@@ -618,8 +619,9 @@ std::optional<StringID> StringInterner::lookup(StringData sd)
 int StringInterner::compare(StringID A, StringID B)
 {
     std::lock_guard lock(m_mutex);
-    REALM_ASSERT_DEBUG(A < m_decompressed_strings.size());
-    REALM_ASSERT_DEBUG(B < m_decompressed_strings.size());
+    // 0 is null, the first index starts from 1.
+    REALM_ASSERT_DEBUG(A <= m_decompressed_strings.size());
+    REALM_ASSERT_DEBUG(B <= m_decompressed_strings.size());
     // comparisons against null
     if (A == B && A == 0)
         return 0;
@@ -635,7 +637,7 @@ int StringInterner::compare(StringID A, StringID B)
 int StringInterner::compare(StringData s, StringID A)
 {
     std::lock_guard lock(m_mutex);
-    REALM_ASSERT_DEBUG(A < m_decompressed_strings.size());
+    REALM_ASSERT_DEBUG(A <= m_decompressed_strings.size());
     // comparisons against null
     if (s.data() == nullptr && A == 0)
         return 0;

test/CMakeLists.txt

@@ -76,6 +76,7 @@ set(CORE_TEST_SOURCES
     test_shared.cpp
     test_status.cpp
     test_string_data.cpp
+    test_string_compression.cpp
     test_table_view.cpp
     test_thread.cpp
     test_transactions.cpp

test/test_group.cpp

@@ -2508,5 +2508,176 @@ TEST(Group_ArrayCompression_Correctness_Random_Input)
 #endif
 }
 
+TEST(Group_ArrayCompression_Strings)
+{
+    GROUP_TEST_PATH(path);
+
+    // create a bunch of string related properties that are going to be compressed and verify write/read machinery
+    // and string correctness.
+    Group to_disk;
+    TableRef table = to_disk.add_table("test");
+    auto col_key_string = table->add_column(type_String, "string");
+    auto col_key_list_string = table->add_column_list(type_String, "list_strings");
+    auto col_key_set_string = table->add_column_set(type_String, "set_strings");
+    auto col_key_dict_string = table->add_column_dictionary(type_String, "dict_strings");
+    auto obj = table->create_object();
+
+
+    obj.set_any(col_key_string, {"Test"});
+    auto list_s = obj.get_list<String>(col_key_list_string);
+    auto set_s = obj.get_set<String>(col_key_set_string);
+    auto dictionary_s = obj.get_dictionary(col_key_dict_string);
+
+    std::string tmp{"aabbbcccaaaaddfwregfgklnjytojfs"};
+    for (size_t i = 0; i < 10; ++i) {
+        list_s.add({tmp + std::to_string(i)});
+    }
+    for (size_t i = 0; i < 10; ++i) {
+        set_s.insert({tmp + std::to_string(i)});
+    }
+    for (size_t i = 0; i < 10; ++i) {
+        const auto key_value = tmp + std::to_string(i);
+        dictionary_s.insert({key_value}, {key_value});
+    }
+
+    CHECK(list_s.size() == 10);
+    CHECK(set_s.size() == 10);
+    CHECK(dictionary_s.size() == 10);
+
+    // Serialize to disk (compression should happen when the proper leaf array is serialized to disk)
+    to_disk.write(path, crypt_key());
+
+#ifdef REALM_DEBUG
+    to_disk.verify();
+#endif
+
+    // Load the tables
+    Group from_disk(path, crypt_key());
+    TableRef read_table = from_disk.get_table("test");
+    auto obj1 = read_table->get_object(0);
+
+    auto list_s1 = obj.get_list<String>("list_strings");
+    auto set_s1 = obj.get_set<String>("set_strings");
+    auto dictionary_s1 = obj.get_dictionary("dict_strings");
+
+    CHECK(obj1.get_any("string") == obj.get_any("string"));
+
+
+    CHECK(list_s1.size() == list_s.size());
+    CHECK(set_s1.size() == set_s.size());
+    CHECK(dictionary_s1.size() == dictionary_s.size());
+
+    CHECK(*read_table == *table);
+
+    for (size_t i = 0; i < list_s1.size(); ++i) {
+        CHECK_EQUAL(list_s1.get_any(i), list_s.get_any(i));
+    }
+
+    for (size_t i = 0; i < set_s1.size(); ++i) {
+        CHECK_EQUAL(set_s1.get_any(i), set_s.get_any(i));
+    }
+
+    for (size_t i = 0; i < dictionary_s1.size(); ++i) {
+        CHECK_EQUAL(dictionary_s1.get_key(i), dictionary_s.get_key(i));
+        CHECK_EQUAL(dictionary_s1.get_any(i), dictionary_s.get_any(i));
+    }
+
+#ifdef REALM_DEBUG
+    from_disk.verify();
+#endif
+}
+
+TEST(Test_Commit_Compression_Strings)
+{
+    auto generate_random_str_len = []() {
+        std::random_device rd;
+        std::mt19937 generator(rd());
+        std::uniform_int_distribution<> distribution(1, 100);
+        return distribution(generator);
+    };
+
+    auto generate_random_string = [](size_t length) {
+        const std::string alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuv"
+                                     "wxyz0123456789";
+        std::random_device rd;
+        std::mt19937 generator(rd());
+        std::uniform_int_distribution<> distribution(0, (int)alphabet.size() - 1);
+
+        std::string random_str;
+        for (size_t i = 0; i < length; ++i)
+            random_str += alphabet[distribution(generator)];
+
+        return random_str;
+    };
+
+    SHARED_GROUP_TEST_PATH(path);
+    auto hist = make_in_realm_history();
+    DBRef db = DB::create(*hist, path);
+    ColKey col_key_string, col_key_list_string, col_key_set_string, col_key_dict_string;
+    ObjKey obj_key;
+    TableKey table_key;
+
+    auto rt = db->start_read();
+    {
+        WriteTransaction wt(db);
+        auto table = wt.add_table("test");
+        table_key = table->get_key();
+        col_key_string = table->add_column(type_String, "string");
+        col_key_list_string = table->add_column_list(type_String, "list_strings");
+        col_key_set_string = table->add_column_set(type_String, "set_strings");
+        col_key_dict_string = table->add_column_dictionary(type_String, "dict_strings");
+        Obj obj = table->create_object();
+        obj_key = obj.get_key();
+        wt.commit();
+    }
+    // check verify that columns have been created
+    rt->advance_read();
+    rt->verify();
+
+    // commit random strings in all the string based columns and verify interner updates
+
+    for (size_t i = 0; i < 50; ++i) {
+
+        // some string
+        const auto str = generate_random_string(generate_random_str_len());
+
+        rt = db->start_read();
+        {
+            WriteTransaction wt(db);
+            auto table = wt.get_table(table_key);
+            auto obj = table->get_object(obj_key);
+
+            obj.set_any(col_key_string, {str});
+            auto list_s = obj.get_list<String>(col_key_list_string);
+            auto set_s = obj.get_set<String>(col_key_set_string);
+            auto dictionary_s = obj.get_dictionary(col_key_dict_string);
+
+            list_s.add({str});
+            set_s.insert({str});
+            dictionary_s.insert({str}, {str});
+
+            wt.commit();
+        }
+        rt->advance_read();
+        rt->verify();
+
+        auto table = rt->get_table(table_key);
+        auto obj = table->get_object(obj_key);
+        const auto current_str = obj.get_any(col_key_string).get_string();
+        CHECK_EQUAL(current_str, str);
+
+        auto list_s = obj.get_list<String>(col_key_list_string);
+        auto set_s = obj.get_set<String>(col_key_set_string);
+        auto dictionary_s = obj.get_dictionary(col_key_dict_string);
+
+        CHECK_EQUAL(list_s.size(), i + 1);
+        CHECK_EQUAL(set_s.size(), i + 1);
+        CHECK_EQUAL(dictionary_s.size(), i + 1);
+
+        CHECK_EQUAL(list_s.get_any(i), str);
+        CHECK_NOT_EQUAL(set_s.find_any(str), not_found);
+        CHECK_NOT_EQUAL(dictionary_s.find_any(str), not_found);
+    }
+}
 
 #endif // TEST_GROUP