Fix reading CT history from synchronized Realms in trawler

tgoyne · tgoyne · commit 5d5e26d6849e · 2024-06-04T14:51:58.000-07:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -20,6 +20,7 @@
 ### Internals
 * Removed references to `stitch_` fields in access tokens in sync unit tests ([PR #7769](https://github.com/realm/realm-core/pull/7769)).
 * Added back iOS simulator testing to evergreen after Jenkins went away ([PR #7758](https://github.com/realm/realm-core/pull/7758)).
+* `realm-trawler -c` did not work on Realm using SyncClient history ([PR #7734](https://github.com/realm/realm-core/pull/7734)).
 
 ----------------------------------------------
 
diff --git a/src/realm/exec/realm_trawler.cpp b/src/realm/exec/realm_trawler.cpp
@@ -65,7 +65,7 @@ void consolidate_lists(std::vector<T>& list, std::vector<T>& list2)
         for (auto it = list.begin() + 1; it != list.end(); ++it) {
             if (prev->start + prev->length != it->start) {
                 if (prev->start + prev->length > it->start) {
-                    std::cout << "*** Overlapping entries:" << std::endl;
+                    std::cout << "*** Overlapping entries:\n";
                     std::cout << std::hex;
                     std::cout << "    0x" << prev->start << "..0x" << prev->start + prev->length << std::endl;
                     std::cout << "    0x" << it->start << "..0x" << it->start + it->length << std::endl;
@@ -455,7 +455,16 @@ class Group : public Array {
             for (size_t n = 0; n < m_history.size(); n++) {
                 ref = m_history.get_ref(n);
                 Node node(m_alloc, ref);
-                ret.emplace_back(node.data(), node.size());
+                if (!node.has_refs()) {
+                    ret.emplace_back(node.data(), node.size());
+                    continue;
+                }
+
+                Array array(m_alloc, ref);
+                for (size_t j = 0; j < array.size(); ++j) {
+                    Node node(m_alloc, array.get_ref(j));
+                    ret.emplace_back(node.data(), node.size());
+                }
             }
         }
         return ret;
@@ -473,7 +482,7 @@ class Group : public Array {
         if (m_evacuation_info.valid()) {
             ostr << "Evacuation limit: " << size_t(m_evacuation_info.get_val(0));
             if (m_evacuation_info.get_val(1)) {
-                ostr << " Scan done" << std::endl;
+                ostr << " Scan done\n";
             }
             else {
                 ostr << " Progress: [";
@@ -482,7 +491,7 @@ class Group : public Array {
                         ostr << ',';
                     ostr << m_evacuation_info.get_val(i);
                 }
-                ostr << "]" << std::endl;
+                ostr << "]\n";
             }
         }
     }
@@ -588,14 +597,14 @@ std::ostream& operator<<(std::ostream& ostr, const Group& g)
         }
     }
     else {
-        ostr << "*** Invalid group ***" << std::endl;
+        ostr << "*** Invalid group ***\n";
     }
     return ostr;
 }
 
 void Table::print_columns(const Group& group) const
 {
-    std::cout << "        <" << m_table_type << ">" << std::endl;
+    std::cout << "        <" << m_table_type << ">\n";
     for (unsigned i = 0; i < m_column_names.size(); i++) {
         auto type = realm::ColumnType(m_column_types.get_val(i) & 0xFFFF);
         auto attr = realm::ColumnAttr(m_column_attributes.get_val(i));
@@ -646,7 +655,7 @@ void Table::print_columns(const Group& group) const
 void Group::print_schema() const
 {
     if (valid()) {
-        std::cout << "Tables: " << std::endl;
+        std::cout << "Tables: \n";
 
         for (unsigned i = 0; i < get_nb_tables(); i++) {
             Table* table = get_table(i);
@@ -808,7 +817,7 @@ void RealmFile::node_scan()
     }
     uint64_t bad_ref = 0;
     if (free_list.empty()) {
-        std::cout << "*** No free list - results may be unreliable ***" << std::endl;
+        std::cout << "*** No free list - results may be unreliable ***\n";
     }
     std::cout << std::hex;
     while (ref < end) {
@@ -847,7 +856,7 @@ void RealmFile::node_scan()
                   << "Start: 0x" << bad_ref << "..0x" << end << std::endl;
     }
     std::cout << std::dec;
-    std::cout << "Allocated space:" << std::endl;
+    std::cout << "Allocated space:\n";
     for (auto s : sizes) {
         std::cout << "    Size: " << s.first << " count: " << s.second << std::endl;
     }
@@ -870,7 +879,7 @@ void RealmFile::memory_leaks()
         consolidate_lists(nodes, free_blocks);
         auto it = nodes.begin();
         if (nodes.size() > 1) {
-            std::cout << "Memory leaked:" << std::endl;
+            std::cout << "Memory leaked:\n";
             auto prev = it;
             ++it;
             while (it != nodes.end()) {
@@ -882,7 +891,7 @@ void RealmFile::memory_leaks()
         }
         else {
             REALM_ASSERT(it->length == m_group->get_file_size());
-            std::cout << "No memory leaks" << std::endl;
+            std::cout << "No memory leaks\n";
         }
     }
 }
@@ -891,7 +900,7 @@ void RealmFile::free_list_info() const
 {
     std::map<uint64_t, unsigned> free_sizes;
     std::map<uint64_t, unsigned> pinned_sizes;
-    std::cout << "Free space:" << std::endl;
+    std::cout << "Free space:\n";
     auto free_list = m_group->get_free_list();
     uint64_t pinned_free_list_size = 0;
     uint64_t total_free_list_size = 0;
@@ -911,11 +920,11 @@ void RealmFile::free_list_info() const
 
         ++it;
     }
-    std::cout << "Free space sizes:" << std::endl;
+    std::cout << "Free space sizes:\n";
     for (auto s : free_sizes) {
         std::cout << "    Size: " << s.first << " count: " << s.second << std::endl;
     }
-    std::cout << "Pinned sizes:" << std::endl;
+    std::cout << "Pinned sizes:\n";
     for (auto s : pinned_sizes) {
         std::cout << "    Size: " << s.first << " count: " << s.second << std::endl;
     }
@@ -984,7 +993,7 @@ class HistoryLogger {
 
     bool dictionary_clear(size_t)
     {
-        std::cout << "Dictionary clear " << std::endl;
+        std::cout << "Dictionary clear \n";
         return true;
     }
 
@@ -1052,8 +1061,13 @@ void RealmFile::changes() const
 
     for (auto c : changesets) {
         realm::util::SimpleInputStream stream(c);
-        parser.parse(stream, logger);
-        std::cout << "--------------------------------------------" << std::endl;
+        try {
+            parser.parse(stream, logger);
+        }
+        catch (const std::exception& ex) {
+            std::cout << "Bad history: " << ex.what() << "\n";
+        }
+        std::cout << "--------------------------------------------\n";
     }
 }
 
@@ -1161,15 +1175,14 @@ int main(int argc, const char* argv[])
         }
     }
     else {
-        std::cout << "Usage: realm-trawler [-afmsw] [--keyfile file-with-binary-crypt-key] [--hexkey "
+        std::cout << "Usage: realm-trawler [-cfmsw] [--keyfile file-with-binary-crypt-key] [--hexkey "
                      "crypt-key-in-hex] [--top "
-                     "top_ref] <realmfile>"
-                  << std::endl;
-        std::cout << "   c : dump changelog" << std::endl;
-        std::cout << "   f : free list analysis" << std::endl;
-        std::cout << "   m : memory leak check" << std::endl;
-        std::cout << "   s : schema dump" << std::endl;
-        std::cout << "   w : node walk" << std::endl;
+                     "top_ref] <realmfile>\n";
+        std::cout << "   c : dump changelog\n";
+        std::cout << "   f : free list analysis\n";
+        std::cout << "   m : memory leak check\n";
+        std::cout << "   s : schema dump\n";
+        std::cout << "   w : node walk\n";
     }
 
     return 0;

Original file line number	Diff line number	Diff line change
`@@ -65,7 +65,7 @@ void consolidate_lists(std::vector<T>& list, std::vector<T>& list2)`
`65`	`65`	`for (auto it = list.begin() + 1; it != list.end(); ++it) {`
`66`	`66`	`if (prev->start + prev->length != it->start) {`
`67`	`67`	`if (prev->start + prev->length > it->start) {`
`68`		`- std::cout << "*** Overlapping entries:" << std::endl;`
	`68`	`+ std::cout << "*** Overlapping entries:\n";`
`69`	`69`	`std::cout << std::hex;`
`70`	`70`	`std::cout << " 0x" << prev->start << "..0x" << prev->start + prev->length << std::endl;`
`71`	`71`	`std::cout << " 0x" << it->start << "..0x" << it->start + it->length << std::endl;`
`@@ -455,7 +455,16 @@ class Group : public Array {`
`455`	`455`	`for (size_t n = 0; n < m_history.size(); n++) {`
`456`	`456`	`ref = m_history.get_ref(n);`
`457`	`457`	`Node node(m_alloc, ref);`
`458`		`- ret.emplace_back(node.data(), node.size());`
	`458`	`+ if (!node.has_refs()) {`
	`459`	`+ ret.emplace_back(node.data(), node.size());`
	`460`	`+ continue;`
	`461`	`+ }`
	`462`	`+`
	`463`	`+ Array array(m_alloc, ref);`
	`464`	`+ for (size_t j = 0; j < array.size(); ++j) {`
	`465`	`+ Node node(m_alloc, array.get_ref(j));`
	`466`	`+ ret.emplace_back(node.data(), node.size());`
	`467`	`+ }`
`459`	`468`	`}`
`460`	`469`	`}`
`461`	`470`	`return ret;`
`@@ -473,7 +482,7 @@ class Group : public Array {`
`473`	`482`	`if (m_evacuation_info.valid()) {`
`474`	`483`	`ostr << "Evacuation limit: " << size_t(m_evacuation_info.get_val(0));`
`475`	`484`	`if (m_evacuation_info.get_val(1)) {`
`476`		`- ostr << " Scan done" << std::endl;`
	`485`	`+ ostr << " Scan done\n";`
`477`	`486`	`}`
`478`	`487`	`else {`
`479`	`488`	`ostr << " Progress: [";`
`@@ -482,7 +491,7 @@ class Group : public Array {`
`482`	`491`	`ostr << ',';`
`483`	`492`	`ostr << m_evacuation_info.get_val(i);`
`484`	`493`	`}`
`485`		`- ostr << "]" << std::endl;`
	`494`	`+ ostr << "]\n";`
`486`	`495`	`}`
`487`	`496`	`}`
`488`	`497`	`}`
`@@ -588,14 +597,14 @@ std::ostream& operator<<(std::ostream& ostr, const Group& g)`
`588`	`597`	`}`
`589`	`598`	`}`
`590`	`599`	`else {`
`591`		`- ostr << "* Invalid group *" << std::endl;`
	`600`	`+ ostr << "* Invalid group *\n";`
`592`	`601`	`}`
`593`	`602`	`return ostr;`
`594`	`603`	`}`
`595`	`604`
`596`	`605`	`void Table::print_columns(const Group& group) const`
`597`	`606`	`{`
`598`		`- std::cout << " <" << m_table_type << ">" << std::endl;`
	`607`	`+ std::cout << " <" << m_table_type << ">\n";`
`599`	`608`	`for (unsigned i = 0; i < m_column_names.size(); i++) {`
`600`	`609`	`auto type = realm::ColumnType(m_column_types.get_val(i) & 0xFFFF);`
`601`	`610`	`auto attr = realm::ColumnAttr(m_column_attributes.get_val(i));`
`@@ -646,7 +655,7 @@ void Table::print_columns(const Group& group) const`
`646`	`655`	`void Group::print_schema() const`
`647`	`656`	`{`
`648`	`657`	`if (valid()) {`
`649`		`- std::cout << "Tables: " << std::endl;`
	`658`	`+ std::cout << "Tables: \n";`
`650`	`659`
`651`	`660`	`for (unsigned i = 0; i < get_nb_tables(); i++) {`
`652`	`661`	`Table* table = get_table(i);`
`@@ -808,7 +817,7 @@ void RealmFile::node_scan()`
`808`	`817`	`}`
`809`	`818`	`uint64_t bad_ref = 0;`
`810`	`819`	`if (free_list.empty()) {`
`811`		`- std::cout << "* No free list - results may be unreliable *" << std::endl;`
	`820`	`+ std::cout << "* No free list - results may be unreliable *\n";`
`812`	`821`	`}`
`813`	`822`	`std::cout << std::hex;`
`814`	`823`	`while (ref < end) {`
`@@ -847,7 +856,7 @@ void RealmFile::node_scan()`
`847`	`856`	`<< "Start: 0x" << bad_ref << "..0x" << end << std::endl;`
`848`	`857`	`}`
`849`	`858`	`std::cout << std::dec;`
`850`		`- std::cout << "Allocated space:" << std::endl;`
	`859`	`+ std::cout << "Allocated space:\n";`
`851`	`860`	`for (auto s : sizes) {`
`852`	`861`	`std::cout << " Size: " << s.first << " count: " << s.second << std::endl;`
`853`	`862`	`}`
`@@ -870,7 +879,7 @@ void RealmFile::memory_leaks()`
`870`	`879`	`consolidate_lists(nodes, free_blocks);`
`871`	`880`	`auto it = nodes.begin();`
`872`	`881`	`if (nodes.size() > 1) {`
`873`		`- std::cout << "Memory leaked:" << std::endl;`
	`882`	`+ std::cout << "Memory leaked:\n";`
`874`	`883`	`auto prev = it;`
`875`	`884`	`++it;`
`876`	`885`	`while (it != nodes.end()) {`
`@@ -882,7 +891,7 @@ void RealmFile::memory_leaks()`
`882`	`891`	`}`
`883`	`892`	`else {`
`884`	`893`	`REALM_ASSERT(it->length == m_group->get_file_size());`
`885`		`- std::cout << "No memory leaks" << std::endl;`
	`894`	`+ std::cout << "No memory leaks\n";`
`886`	`895`	`}`
`887`	`896`	`}`
`888`	`897`	`}`
`@@ -891,7 +900,7 @@ void RealmFile::free_list_info() const`
`891`	`900`	`{`
`892`	`901`	`std::map<uint64_t, unsigned> free_sizes;`
`893`	`902`	`std::map<uint64_t, unsigned> pinned_sizes;`
`894`		`- std::cout << "Free space:" << std::endl;`
	`903`	`+ std::cout << "Free space:\n";`
`895`	`904`	`auto free_list = m_group->get_free_list();`
`896`	`905`	`uint64_t pinned_free_list_size = 0;`
`897`	`906`	`uint64_t total_free_list_size = 0;`
`@@ -911,11 +920,11 @@ void RealmFile::free_list_info() const`
`911`	`920`
`912`	`921`	`++it;`
`913`	`922`	`}`
`914`		`- std::cout << "Free space sizes:" << std::endl;`
	`923`	`+ std::cout << "Free space sizes:\n";`
`915`	`924`	`for (auto s : free_sizes) {`
`916`	`925`	`std::cout << " Size: " << s.first << " count: " << s.second << std::endl;`
`917`	`926`	`}`
`918`		`- std::cout << "Pinned sizes:" << std::endl;`
	`927`	`+ std::cout << "Pinned sizes:\n";`
`919`	`928`	`for (auto s : pinned_sizes) {`
`920`	`929`	`std::cout << " Size: " << s.first << " count: " << s.second << std::endl;`
`921`	`930`	`}`
`@@ -984,7 +993,7 @@ class HistoryLogger {`
`984`	`993`
`985`	`994`	`bool dictionary_clear(size_t)`
`986`	`995`	`{`
`987`		`- std::cout << "Dictionary clear " << std::endl;`
	`996`	`+ std::cout << "Dictionary clear \n";`
`988`	`997`	`return true;`
`989`	`998`	`}`
`990`	`999`
`@@ -1052,8 +1061,13 @@ void RealmFile::changes() const`
`1052`	`1061`
`1053`	`1062`	`for (auto c : changesets) {`
`1054`	`1063`	`realm::util::SimpleInputStream stream(c);`
`1055`		`- parser.parse(stream, logger);`
`1056`		`- std::cout << "--------------------------------------------" << std::endl;`
	`1064`	`+ try {`
	`1065`	`+ parser.parse(stream, logger);`
	`1066`	`+ }`
	`1067`	`+ catch (const std::exception& ex) {`
	`1068`	`+ std::cout << "Bad history: " << ex.what() << "\n";`
	`1069`	`+ }`
	`1070`	`+ std::cout << "--------------------------------------------\n";`
`1057`	`1071`	`}`
`1058`	`1072`	`}`
`1059`	`1073`
`@@ -1161,15 +1175,14 @@ int main(int argc, const char* argv[])`
`1161`	`1175`	`}`
`1162`	`1176`	`}`
`1163`	`1177`	`else {`
`1164`		`- std::cout << "Usage: realm-trawler [-afmsw] [--keyfile file-with-binary-crypt-key] [--hexkey "`
	`1178`	`+ std::cout << "Usage: realm-trawler [-cfmsw] [--keyfile file-with-binary-crypt-key] [--hexkey "`
`1165`	`1179`	`"crypt-key-in-hex] [--top "`
`1166`		`- "top_ref] <realmfile>"`
`1167`		`- << std::endl;`
`1168`		`- std::cout << " c : dump changelog" << std::endl;`
`1169`		`- std::cout << " f : free list analysis" << std::endl;`
`1170`		`- std::cout << " m : memory leak check" << std::endl;`
`1171`		`- std::cout << " s : schema dump" << std::endl;`
`1172`		`- std::cout << " w : node walk" << std::endl;`
	`1180`	`+ "top_ref] <realmfile>\n";`
	`1181`	`+ std::cout << " c : dump changelog\n";`
	`1182`	`+ std::cout << " f : free list analysis\n";`
	`1183`	`+ std::cout << " m : memory leak check\n";`
	`1184`	`+ std::cout << " s : schema dump\n";`
	`1185`	`+ std::cout << " w : node walk\n";`
`1173`	`1186`	`}`
`1174`	`1187`
`1175`	`1188`	`return 0;`