Merge branch 'master' of github.com:realm/realm-core into next-major

Author: nicola-cab

CHANGELOG.md

@@ -21,15 +21,17 @@
 
 ----------------------------------------------
 
-# NEXT RELEASE
+# 14.5.0 Release notes
 
 ### Enhancements
 * Introduce sync 'progress_estimate' parameter (value from 0.0 to 1.0) for existing sync 'ProgressNotifierCallback' api to report sync progress on current batch of upload/download until completion ([#7450](https://github.com/realm/realm-core/issues/7450))
 
 ### Fixed
-* <How do the end-user experience this issue? what was the impact?> ([#????](https://github.com/realm/realm-core/issues/????), since v?.?.?)
 * Fix an assertion failure "m_lock_info && m_lock_info->m_file.get_path() == m_filename" that appears to be related to opening a Realm while the file is in the process of being closed on another thread ([Swift #8507](https://github.com/realm/realm-swift/issues/8507)).
 * Fixed diverging history due to a bug in the replication code when setting default null values (embedded objects included) ([#7536](https://github.com/realm/realm-core/issues/7536)).
+* Version 19.39.33523 of MSVC would crash when compiling for arm64 in release mode ([PR #7533](https://github.com/realm/realm-core/pull/7533)).
+* Null pointer exception may be triggered when logging out and async commits callbacks not executed ([#7434](https://github.com/realm/realm-core/issues/7434), since v13.26.0)
+* Fixed building for iPhone simulators targeting deployment target 11 ([#7554](https://github.com/realm/realm-core/pull/7554)).
 
 ### Breaking changes
 * Updated default base URL to be `https://services.cloud.mongodb.com` to support the new domains (was `https://realm.mongodb.com`). ([PR #7534](https://github.com/realm/realm-core/pull/7534))
@@ -42,7 +44,10 @@
 ### Internals
 * Update libuv used in object store tests from v1.35.0 to v1.48.0 ([PR #7508](https://github.com/realm/realm-core/pull/7508)).
 * Made `set_default_logger` nullable in the bindgen spec.yml (PR [#7515](https://github.com/realm/realm-core/pull/7515)).
+* Recreating the sync metadata Realm when the encryption key changes is now done in a multi-process safe manner ([PR #7526](https://github.com/realm/realm-core/pull/7526)).
 * Added `App::default_base_url()` static accessor for SDKs to retrieve the default base URL from Core. ([PR #7534](https://github.com/realm/realm-core/pull/7534))
+* Realm2JSON tool will now correctly upgrade file to current fileformat.
+* (bindgen) Remove dependency on the `clang-format` package and rely on a binary provided by the system instead.
 
 ----------------------------------------------
 

Package.swift

@@ -3,7 +3,7 @@
 import PackageDescription
 import Foundation
 
-let versionStr = "14.4.1"
+let versionStr = "14.5.0"
 let versionPieces = versionStr.split(separator: "-")
 let versionCompontents = versionPieces[0].split(separator: ".")
 let versionExtra = versionPieces.count > 1 ? versionPieces[1] : ""

README.md

@@ -20,7 +20,7 @@ This repository holds the source code for the core database component used by al
 
 Realm Core is not in itself an "end-user" product with a publicly stable and supported API.
 
-Refer to the [Realm documentation](https://www.mongodb.com/docs/realm) for information about Realm and using the SDKs.
+Refer to the [Atlas Device SDK documentation](https://www.mongodb.com/docs/atlas/device-sdks/) for information about Realm and using the SDKs.
 
 ## Building Realm
 

bindgen/src/formatter.ts

@@ -67,4 +67,4 @@ export function format(formatter: Formatter, cwd: string, filePaths: string[]):
   }
 }
 
-export const clangFormat = createCommandFormatter("clang", ["npx", "clang-format", "-i"]);
+export const clangFormat = createCommandFormatter("clang", ["clang-format", "-i"]);

dependencies.yml

@@ -1,5 +1,5 @@
 PACKAGE_NAME: realm-core
-VERSION: 14.4.1
+VERSION: 14.5.0
 OPENSSL_VERSION: 3.2.0
 ZLIB_VERSION: 1.2.13
 # https://github.com/10gen/baas/commits

package.json

@@ -22,7 +22,6 @@
     "chalk": "^4.1.2",
     "change-case": "^4.1.2",
     "chevrotain": "^10.4.0",
-    "clang-format": "^1.8.0",
     "commander": "^11.1.0",
     "debug": "^4.3.4",
     "typescript-json-schema": "^0.55.0",

src/realm/alloc_slab.cpp

@@ -813,17 +813,39 @@ ref_type SlabAlloc::attach_file(const std::string& path, Config& cfg, util::Writ
     // the call below to set_encryption_key.
     m_file.set_encryption_key(cfg.encryption_key);
 
+    note_reader_start(this);
+    util::ScopeExit reader_end_guard([this]() noexcept {
+        note_reader_end(this);
+    });
     size_t size = 0;
     // The size of a database file must not exceed what can be encoded in
     // size_t.
     if (REALM_UNLIKELY(int_cast_with_overflow_detect(m_file.get_size(), size)))
         throw InvalidDatabase("Realm file too large", path);
-    if (cfg.encryption_key && size == 0 && physical_file_size != 0) {
+    if (cfg.clear_file_on_error && cfg.session_initiator) {
+        if (size == 0 && physical_file_size != 0) {
+            cfg.clear_file = true;
+        }
+        else if (size > 0) {
+            try {
+                read_and_validate_header(m_file, path, size, cfg.session_initiator, m_write_observer);
+            }
+            catch (const InvalidDatabase&) {
+                cfg.clear_file = true;
+            }
+        }
+    }
+    if (cfg.clear_file) {
+        m_file.resize(0);
+        size = 0;
+        physical_file_size = 0;
+    }
+    else if (cfg.encryption_key && !cfg.clear_file && size == 0 && physical_file_size != 0) {
         // The opened file holds data, but is so small it cannot have
         // been created with encryption
         throw InvalidDatabase("Attempt to open unencrypted file with encryption key", path);
     }
-    if (size == 0 || cfg.clear_file) {
+    if (size == 0) {
         if (REALM_UNLIKELY(cfg.read_only))
             throw InvalidDatabase("Read-only access to empty Realm file", path);
 
@@ -833,7 +855,7 @@ ref_type SlabAlloc::attach_file(const std::string& path, Config& cfg, util::Writ
         // mappings_for_file in the encryption layer. So the prealloc() is required before
         // interacting with the encryption layer in File::write().
         // Pre-alloc initial space
-        m_file.prealloc(initial_size);     // Throws
+        m_file.prealloc(initial_size); // Throws
         // seek() back to the start of the file in preparation for writing the header
         // This sequence of File operations is protected from races by
         // DB::m_controlmutex, so we know we are the only ones operating on the file
@@ -847,65 +869,9 @@ ref_type SlabAlloc::attach_file(const std::string& path, Config& cfg, util::Writ
 
         size = initial_size;
     }
-    ref_type top_ref;
-    note_reader_start(this);
-    util::ScopeExit reader_end_guard([this]() noexcept {
-        note_reader_end(this);
-    });
-
-    try {
-        // we'll read header and (potentially) footer
-        File::Map<char> map_header(m_file, File::access_ReadOnly, sizeof(Header), 0, m_write_observer);
-        realm::util::encryption_read_barrier(map_header, 0, sizeof(Header));
-        auto header = reinterpret_cast<const Header*>(map_header.get_addr());
 
-        File::Map<char> map_footer;
-        const StreamingFooter* footer = nullptr;
-        if (is_file_on_streaming_form(*header) && size >= sizeof(StreamingFooter) + sizeof(Header)) {
-            size_t footer_ref = size - sizeof(StreamingFooter);
-            size_t footer_page_base = footer_ref & ~(page_size() - 1);
-            size_t footer_offset = footer_ref - footer_page_base;
-            map_footer = File::Map<char>(m_file, footer_page_base, File::access_ReadOnly,
-                                         sizeof(StreamingFooter) + footer_offset, 0, m_write_observer);
-            realm::util::encryption_read_barrier(map_footer, footer_offset, sizeof(StreamingFooter));
-            footer = reinterpret_cast<const StreamingFooter*>(map_footer.get_addr() + footer_offset);
-        }
-
-        top_ref = validate_header(header, footer, size, path, cfg.encryption_key != nullptr); // Throws
-        m_attach_mode = cfg.is_shared ? attach_SharedFile : attach_UnsharedFile;
-        m_data = map_header.get_addr(); // <-- needed below
-
-        if (cfg.session_initiator && is_file_on_streaming_form(*header)) {
-            // Don't compare file format version fields as they are allowed to differ.
-            // Also don't compare reserved fields.
-            REALM_ASSERT_EX(header->m_flags == 0, header->m_flags, get_file_path_for_assertions());
-            REALM_ASSERT_EX(header->m_mnemonic[0] == uint8_t('T'), header->m_mnemonic[0],
-                            get_file_path_for_assertions());
-            REALM_ASSERT_EX(header->m_mnemonic[1] == uint8_t('-'), header->m_mnemonic[1],
-                            get_file_path_for_assertions());
-            REALM_ASSERT_EX(header->m_mnemonic[2] == uint8_t('D'), header->m_mnemonic[2],
-                            get_file_path_for_assertions());
-            REALM_ASSERT_EX(header->m_mnemonic[3] == uint8_t('B'), header->m_mnemonic[3],
-                            get_file_path_for_assertions());
-            REALM_ASSERT_EX(header->m_top_ref[0] == 0xFFFFFFFFFFFFFFFFULL, header->m_top_ref[0],
-                            get_file_path_for_assertions());
-            REALM_ASSERT_EX(header->m_top_ref[1] == 0, header->m_top_ref[1], get_file_path_for_assertions());
-            REALM_ASSERT_EX(footer->m_magic_cookie == footer_magic_cookie, footer->m_magic_cookie,
-                            get_file_path_for_assertions());
-        }
-    }
-    catch (const InvalidDatabase&) {
-        throw;
-    }
-    catch (const DecryptionFailed& e) {
-        throw InvalidDatabase(util::format("Realm file decryption failed (%1)", e.what()), path);
-    }
-    catch (const std::exception& e) {
-        throw InvalidDatabase(e.what(), path);
-    }
-    catch (...) {
-        throw InvalidDatabase("unknown error", path);
-    }
+    ref_type top_ref = read_and_validate_header(m_file, path, size, cfg.session_initiator, m_write_observer);
+    m_attach_mode = cfg.is_shared ? attach_SharedFile : attach_UnsharedFile;
     // m_data not valid at this point!
     m_baseline = 0;
     // make sure that any call to begin_read cause any slab to be placed in free
@@ -1040,6 +1006,57 @@ void SlabAlloc::attach_empty()
     m_ref_translation_ptr = new RefTranslation[1];
 }
 
+ref_type SlabAlloc::read_and_validate_header(util::File& file, const std::string& path, size_t size,
+                                             bool session_initiator, util::WriteObserver* write_observer)
+{
+    try {
+        // we'll read header and (potentially) footer
+        File::Map<char> map_header(file, File::access_ReadOnly, sizeof(Header), 0, write_observer);
+        realm::util::encryption_read_barrier(map_header, 0, sizeof(Header));
+        auto header = reinterpret_cast<const Header*>(map_header.get_addr());
+
+        File::Map<char> map_footer;
+        const StreamingFooter* footer = nullptr;
+        if (is_file_on_streaming_form(*header) && size >= sizeof(StreamingFooter) + sizeof(Header)) {
+            size_t footer_ref = size - sizeof(StreamingFooter);
+            size_t footer_page_base = footer_ref & ~(page_size() - 1);
+            size_t footer_offset = footer_ref - footer_page_base;
+            map_footer = File::Map<char>(file, footer_page_base, File::access_ReadOnly,
+                                         sizeof(StreamingFooter) + footer_offset, 0, write_observer);
+            realm::util::encryption_read_barrier(map_footer, footer_offset, sizeof(StreamingFooter));
+            footer = reinterpret_cast<const StreamingFooter*>(map_footer.get_addr() + footer_offset);
+        }
+
+        auto top_ref = validate_header(header, footer, size, path, file.get_encryption_key() != nullptr); // Throws
+
+        if (session_initiator && is_file_on_streaming_form(*header)) {
+            // Don't compare file format version fields as they are allowed to differ.
+            // Also don't compare reserved fields.
+            REALM_ASSERT_EX(header->m_flags == 0, header->m_flags, path);
+            REALM_ASSERT_EX(header->m_mnemonic[0] == uint8_t('T'), header->m_mnemonic[0], path);
+            REALM_ASSERT_EX(header->m_mnemonic[1] == uint8_t('-'), header->m_mnemonic[1], path);
+            REALM_ASSERT_EX(header->m_mnemonic[2] == uint8_t('D'), header->m_mnemonic[2], path);
+            REALM_ASSERT_EX(header->m_mnemonic[3] == uint8_t('B'), header->m_mnemonic[3], path);
+            REALM_ASSERT_EX(header->m_top_ref[0] == 0xFFFFFFFFFFFFFFFFULL, header->m_top_ref[0], path);
+            REALM_ASSERT_EX(header->m_top_ref[1] == 0, header->m_top_ref[1], path);
+            REALM_ASSERT_EX(footer->m_magic_cookie == footer_magic_cookie, footer->m_magic_cookie, path);
+        }
+        return top_ref;
+    }
+    catch (const InvalidDatabase&) {
+        throw;
+    }
+    catch (const DecryptionFailed& e) {
+        throw InvalidDatabase(util::format("Realm file decryption failed (%1)", e.what()), path);
+    }
+    catch (const std::exception& e) {
+        throw InvalidDatabase(e.what(), path);
+    }
+    catch (...) {
+        throw InvalidDatabase("unknown error", path);
+    }
+}
+
 void SlabAlloc::throw_header_exception(std::string msg, const Header& header, const std::string& path)
 {
     char buf[256];

src/realm/alloc_slab.hpp

@@ -108,15 +108,20 @@ class SlabAlloc : public Allocator {
     /// Always initialize the file as if it was a newly
     /// created file and ignore any pre-existing contents. Requires that
     /// Config::session_initiator be true as well.
+    ///
+    /// \var Config::clear_file_on_error
+    /// If the file being opened is not a valid Realm file (possibly due to a
+    /// decryption failure), reinitialize it as if clear_file was set.
     struct Config {
+        const char* encryption_key = nullptr;
         bool is_shared = false;
         bool read_only = false;
         bool no_create = false;
         bool skip_validate = false;
         bool session_initiator = false;
         bool clear_file = false;
+        bool clear_file_on_error = false;
         bool disable_sync = false;
-        const char* encryption_key = nullptr;
     };
 
     struct Retry {};
@@ -363,6 +368,11 @@ class SlabAlloc : public Allocator {
     void note_reader_start(const void* reader_id);
     void note_reader_end(const void* reader_id) noexcept;
 
+    /// Read the header (and possibly footer) from the file, returning the top ref if it's valid and throwing
+    /// InvalidDatabase otherwise.
+    static ref_type read_and_validate_header(util::File& file, const std::string& path, size_t size,
+                                             bool session_initiator, util::WriteObserver* write_observer);
+
     void verify() const override;
 #ifdef REALM_DEBUG
     void enable_debug(bool enable)
@@ -703,10 +713,10 @@ class SlabAlloc : public Allocator {
     /// corrupted, or if the specified encryption key is incorrect. This
     /// function will not detect all forms of corruption, though.
     /// Returns the top_ref for the latest commit.
-    ref_type validate_header(const char* data, size_t len, const std::string& path);
-    ref_type validate_header(const Header* header, const StreamingFooter* footer, size_t size,
-                             const std::string& path, bool is_encrypted = false);
-    void throw_header_exception(std::string msg, const Header& header, const std::string& path);
+    static ref_type validate_header(const char* data, size_t len, const std::string& path);
+    static ref_type validate_header(const Header* header, const StreamingFooter* footer, size_t size,
+                                    const std::string& path, bool is_encrypted = false);
+    static void throw_header_exception(std::string msg, const Header& header, const std::string& path);
 
     static bool is_file_on_streaming_form(const Header& header);
     /// Read the top_ref from the given buffer and set m_file_on_streaming_form

src/realm/db.cpp

@@ -898,7 +898,7 @@ std::string DBOptions::sys_tmp_dir = getenv("TMPDIR") ? getenv("TMPDIR") : "";
 // initializing process crashes and leaves the shared memory in an
 // undefined state.
 
-void DB::open(const std::string& path, bool no_create_file, const DBOptions& options)
+void DB::open(const std::string& path, const DBOptions& options)
 {
     // Exception safety: Since do_open() is called from constructors, if it
     // throws, it must leave the file closed.
@@ -1144,10 +1144,11 @@ void DB::open(const std::string& path, bool no_create_file, const DBOptions& opt
             cfg.read_only = false;
             cfg.skip_validate = !begin_new_session;
             cfg.disable_sync = options.durability == Durability::MemOnly || options.durability == Durability::Unsafe;
+            cfg.clear_file_on_error = options.clear_on_invalid_file;
 
             // only the session initiator is allowed to create the database, all other
             // must assume that it already exists.
-            cfg.no_create = (begin_new_session ? no_create_file : true);
+            cfg.no_create = (begin_new_session ? options.no_create : true);
 
             // if we're opening a MemOnly file that isn't already opened by
             // someone else then it's a file which should have been deleted on
@@ -1499,8 +1500,7 @@ void DB::open(Replication& repl, const std::string& file, const DBOptions& optio
 
     set_replication(&repl);
 
-    bool no_create = false;
-    open(file, no_create, options); // Throws
+    open(file, options); // Throws
 }
 
 class DBLogger : public Logger {
@@ -1532,7 +1532,7 @@ void DB::set_logger(const std::shared_ptr<util::Logger>& logger) noexcept
         m_logger = std::make_shared<DBLogger>(logger, m_log_id);
 }
 
-void DB::open(Replication& repl, const DBOptions options)
+void DB::open(Replication& repl, const DBOptions& options)
 {
     REALM_ASSERT(!is_attached());
     repl.initialize(*this); // Throws
@@ -2808,10 +2808,10 @@ inline DB::DB(Private, const DBOptions& options)
     }
 }
 
-DBRef DB::create(const std::string& file, bool no_create, const DBOptions& options) NO_THREAD_SAFETY_ANALYSIS
+DBRef DB::create(const std::string& file, const DBOptions& options) NO_THREAD_SAFETY_ANALYSIS
 {
     DBRef retval = std::make_shared<DB>(Private(), options);
-    retval->open(file, no_create, options);
+    retval->open(file, options);
     return retval;
 }