Truncate the metadata realm in-place rather than deleting and recreating it (#7526)

This solves some locking problems in multi-process scenarios. The flow of
checking if the key is valid, deleting the file if not, and then recreating the
file if needed needs to be done with an exclusive lock on the file held. This
can't be done with DB::call_with_lock() because there isn't a good way to
initialize the new file from within the call_with_lock() callback, and instead
needs to be pushed into the file initialization guarded by the control mutex.

Co-authored-by: Finn Schiermer Andersen <[email protected]>

Author: tgoyne

CHANGELOG.md

@@ -19,6 +19,7 @@
 ### Internals
 * Update libuv used in object store tests from v1.35.0 to v1.48.0 ([PR #7508](https://github.com/realm/realm-core/pull/7508)).
 * Made `set_default_logger` nullable in the bindgen spec.yml (PR [#7515](https://github.com/realm/realm-core/pull/7515)).
+* Recreating the sync metadata Realm when the encryption key changes is now done in a multi-process safe manner ([PR #7526](https://github.com/realm/realm-core/pull/7526)).
 * Added `App::default_base_url()` static accessor for SDKs to retrieve the default base URL from Core. ([PR #7534](https://github.com/realm/realm-core/pull/7534))
 * Realm2JSON tool will now correctly upgrade file to current fileformat.
 

src/realm/alloc_slab.cpp

@@ -813,17 +813,39 @@ ref_type SlabAlloc::attach_file(const std::string& path, Config& cfg, util::Writ
     // the call below to set_encryption_key.
     m_file.set_encryption_key(cfg.encryption_key);
 
+    note_reader_start(this);
+    util::ScopeExit reader_end_guard([this]() noexcept {
+        note_reader_end(this);
+    });
     size_t size = 0;
     // The size of a database file must not exceed what can be encoded in
     // size_t.
     if (REALM_UNLIKELY(int_cast_with_overflow_detect(m_file.get_size(), size)))
         throw InvalidDatabase("Realm file too large", path);
-    if (cfg.encryption_key && size == 0 && physical_file_size != 0) {
+    if (cfg.clear_file_on_error && cfg.session_initiator) {
+        if (size == 0 && physical_file_size != 0) {
+            cfg.clear_file = true;
+        }
+        else if (size > 0) {
+            try {
+                read_and_validate_header(m_file, path, size, cfg.session_initiator, m_write_observer);
+            }
+            catch (const InvalidDatabase&) {
+                cfg.clear_file = true;
+            }
+        }
+    }
+    if (cfg.clear_file) {
+        m_file.resize(0);
+        size = 0;
+        physical_file_size = 0;
+    }
+    else if (cfg.encryption_key && !cfg.clear_file && size == 0 && physical_file_size != 0) {
         // The opened file holds data, but is so small it cannot have
         // been created with encryption
         throw InvalidDatabase("Attempt to open unencrypted file with encryption key", path);
     }
-    if (size == 0 || cfg.clear_file) {
+    if (size == 0) {
         if (REALM_UNLIKELY(cfg.read_only))
             throw InvalidDatabase("Read-only access to empty Realm file", path);
 
@@ -833,7 +855,7 @@ ref_type SlabAlloc::attach_file(const std::string& path, Config& cfg, util::Writ
         // mappings_for_file in the encryption layer. So the prealloc() is required before
         // interacting with the encryption layer in File::write().
         // Pre-alloc initial space
-        m_file.prealloc(initial_size);     // Throws
+        m_file.prealloc(initial_size); // Throws
         // seek() back to the start of the file in preparation for writing the header
         // This sequence of File operations is protected from races by
         // DB::m_controlmutex, so we know we are the only ones operating on the file
@@ -847,65 +869,9 @@ ref_type SlabAlloc::attach_file(const std::string& path, Config& cfg, util::Writ
 
         size = initial_size;
     }
-    ref_type top_ref;
-    note_reader_start(this);
-    util::ScopeExit reader_end_guard([this]() noexcept {
-        note_reader_end(this);
-    });
-
-    try {
-        // we'll read header and (potentially) footer
-        File::Map<char> map_header(m_file, File::access_ReadOnly, sizeof(Header), 0, m_write_observer);
-        realm::util::encryption_read_barrier(map_header, 0, sizeof(Header));
-        auto header = reinterpret_cast<const Header*>(map_header.get_addr());
 
-        File::Map<char> map_footer;
-        const StreamingFooter* footer = nullptr;
-        if (is_file_on_streaming_form(*header) && size >= sizeof(StreamingFooter) + sizeof(Header)) {
-            size_t footer_ref = size - sizeof(StreamingFooter);
-            size_t footer_page_base = footer_ref & ~(page_size() - 1);
-            size_t footer_offset = footer_ref - footer_page_base;
-            map_footer = File::Map<char>(m_file, footer_page_base, File::access_ReadOnly,
-                                         sizeof(StreamingFooter) + footer_offset, 0, m_write_observer);
-            realm::util::encryption_read_barrier(map_footer, footer_offset, sizeof(StreamingFooter));
-            footer = reinterpret_cast<const StreamingFooter*>(map_footer.get_addr() + footer_offset);
-        }
-
-        top_ref = validate_header(header, footer, size, path, cfg.encryption_key != nullptr); // Throws
-        m_attach_mode = cfg.is_shared ? attach_SharedFile : attach_UnsharedFile;
-        m_data = map_header.get_addr(); // <-- needed below
-
-        if (cfg.session_initiator && is_file_on_streaming_form(*header)) {
-            // Don't compare file format version fields as they are allowed to differ.
-            // Also don't compare reserved fields.
-            REALM_ASSERT_EX(header->m_flags == 0, header->m_flags, get_file_path_for_assertions());
-            REALM_ASSERT_EX(header->m_mnemonic[0] == uint8_t('T'), header->m_mnemonic[0],
-                            get_file_path_for_assertions());
-            REALM_ASSERT_EX(header->m_mnemonic[1] == uint8_t('-'), header->m_mnemonic[1],
-                            get_file_path_for_assertions());
-            REALM_ASSERT_EX(header->m_mnemonic[2] == uint8_t('D'), header->m_mnemonic[2],
-                            get_file_path_for_assertions());
-            REALM_ASSERT_EX(header->m_mnemonic[3] == uint8_t('B'), header->m_mnemonic[3],
-                            get_file_path_for_assertions());
-            REALM_ASSERT_EX(header->m_top_ref[0] == 0xFFFFFFFFFFFFFFFFULL, header->m_top_ref[0],
-                            get_file_path_for_assertions());
-            REALM_ASSERT_EX(header->m_top_ref[1] == 0, header->m_top_ref[1], get_file_path_for_assertions());
-            REALM_ASSERT_EX(footer->m_magic_cookie == footer_magic_cookie, footer->m_magic_cookie,
-                            get_file_path_for_assertions());
-        }
-    }
-    catch (const InvalidDatabase&) {
-        throw;
-    }
-    catch (const DecryptionFailed& e) {
-        throw InvalidDatabase(util::format("Realm file decryption failed (%1)", e.what()), path);
-    }
-    catch (const std::exception& e) {
-        throw InvalidDatabase(e.what(), path);
-    }
-    catch (...) {
-        throw InvalidDatabase("unknown error", path);
-    }
+    ref_type top_ref = read_and_validate_header(m_file, path, size, cfg.session_initiator, m_write_observer);
+    m_attach_mode = cfg.is_shared ? attach_SharedFile : attach_UnsharedFile;
     // m_data not valid at this point!
     m_baseline = 0;
     // make sure that any call to begin_read cause any slab to be placed in free
@@ -1040,6 +1006,57 @@ void SlabAlloc::attach_empty()
     m_ref_translation_ptr = new RefTranslation[1];
 }
 
+ref_type SlabAlloc::read_and_validate_header(util::File& file, const std::string& path, size_t size,
+                                             bool session_initiator, util::WriteObserver* write_observer)
+{
+    try {
+        // we'll read header and (potentially) footer
+        File::Map<char> map_header(file, File::access_ReadOnly, sizeof(Header), 0, write_observer);
+        realm::util::encryption_read_barrier(map_header, 0, sizeof(Header));
+        auto header = reinterpret_cast<const Header*>(map_header.get_addr());
+
+        File::Map<char> map_footer;
+        const StreamingFooter* footer = nullptr;
+        if (is_file_on_streaming_form(*header) && size >= sizeof(StreamingFooter) + sizeof(Header)) {
+            size_t footer_ref = size - sizeof(StreamingFooter);
+            size_t footer_page_base = footer_ref & ~(page_size() - 1);
+            size_t footer_offset = footer_ref - footer_page_base;
+            map_footer = File::Map<char>(file, footer_page_base, File::access_ReadOnly,
+                                         sizeof(StreamingFooter) + footer_offset, 0, write_observer);
+            realm::util::encryption_read_barrier(map_footer, footer_offset, sizeof(StreamingFooter));
+            footer = reinterpret_cast<const StreamingFooter*>(map_footer.get_addr() + footer_offset);
+        }
+
+        auto top_ref = validate_header(header, footer, size, path, file.get_encryption_key() != nullptr); // Throws
+
+        if (session_initiator && is_file_on_streaming_form(*header)) {
+            // Don't compare file format version fields as they are allowed to differ.
+            // Also don't compare reserved fields.
+            REALM_ASSERT_EX(header->m_flags == 0, header->m_flags, path);
+            REALM_ASSERT_EX(header->m_mnemonic[0] == uint8_t('T'), header->m_mnemonic[0], path);
+            REALM_ASSERT_EX(header->m_mnemonic[1] == uint8_t('-'), header->m_mnemonic[1], path);
+            REALM_ASSERT_EX(header->m_mnemonic[2] == uint8_t('D'), header->m_mnemonic[2], path);
+            REALM_ASSERT_EX(header->m_mnemonic[3] == uint8_t('B'), header->m_mnemonic[3], path);
+            REALM_ASSERT_EX(header->m_top_ref[0] == 0xFFFFFFFFFFFFFFFFULL, header->m_top_ref[0], path);
+            REALM_ASSERT_EX(header->m_top_ref[1] == 0, header->m_top_ref[1], path);
+            REALM_ASSERT_EX(footer->m_magic_cookie == footer_magic_cookie, footer->m_magic_cookie, path);
+        }
+        return top_ref;
+    }
+    catch (const InvalidDatabase&) {
+        throw;
+    }
+    catch (const DecryptionFailed& e) {
+        throw InvalidDatabase(util::format("Realm file decryption failed (%1)", e.what()), path);
+    }
+    catch (const std::exception& e) {
+        throw InvalidDatabase(e.what(), path);
+    }
+    catch (...) {
+        throw InvalidDatabase("unknown error", path);
+    }
+}
+
 void SlabAlloc::throw_header_exception(std::string msg, const Header& header, const std::string& path)
 {
     char buf[256];

src/realm/alloc_slab.hpp

@@ -108,15 +108,20 @@ class SlabAlloc : public Allocator {
     /// Always initialize the file as if it was a newly
     /// created file and ignore any pre-existing contents. Requires that
     /// Config::session_initiator be true as well.
+    ///
+    /// \var Config::clear_file_on_error
+    /// If the file being opened is not a valid Realm file (possibly due to a
+    /// decryption failure), reinitialize it as if clear_file was set.
     struct Config {
+        const char* encryption_key = nullptr;
         bool is_shared = false;
         bool read_only = false;
         bool no_create = false;
         bool skip_validate = false;
         bool session_initiator = false;
         bool clear_file = false;
+        bool clear_file_on_error = false;
         bool disable_sync = false;
-        const char* encryption_key = nullptr;
     };
 
     struct Retry {};
@@ -363,6 +368,11 @@ class SlabAlloc : public Allocator {
     void note_reader_start(const void* reader_id);
     void note_reader_end(const void* reader_id) noexcept;
 
+    /// Read the header (and possibly footer) from the file, returning the top ref if it's valid and throwing
+    /// InvalidDatabase otherwise.
+    static ref_type read_and_validate_header(util::File& file, const std::string& path, size_t size,
+                                             bool session_initiator, util::WriteObserver* write_observer);
+
     void verify() const override;
 #ifdef REALM_DEBUG
     void enable_debug(bool enable)
@@ -703,10 +713,10 @@ class SlabAlloc : public Allocator {
     /// corrupted, or if the specified encryption key is incorrect. This
     /// function will not detect all forms of corruption, though.
     /// Returns the top_ref for the latest commit.
-    ref_type validate_header(const char* data, size_t len, const std::string& path);
-    ref_type validate_header(const Header* header, const StreamingFooter* footer, size_t size,
-                             const std::string& path, bool is_encrypted = false);
-    void throw_header_exception(std::string msg, const Header& header, const std::string& path);
+    static ref_type validate_header(const char* data, size_t len, const std::string& path);
+    static ref_type validate_header(const Header* header, const StreamingFooter* footer, size_t size,
+                                    const std::string& path, bool is_encrypted = false);
+    static void throw_header_exception(std::string msg, const Header& header, const std::string& path);
 
     static bool is_file_on_streaming_form(const Header& header);
     /// Read the top_ref from the given buffer and set m_file_on_streaming_form

src/realm/db.cpp

@@ -898,7 +898,7 @@ std::string DBOptions::sys_tmp_dir = getenv("TMPDIR") ? getenv("TMPDIR") : "";
 // initializing process crashes and leaves the shared memory in an
 // undefined state.
 
-void DB::open(const std::string& path, bool no_create_file, const DBOptions& options)
+void DB::open(const std::string& path, const DBOptions& options)
 {
     // Exception safety: Since do_open() is called from constructors, if it
     // throws, it must leave the file closed.
@@ -1144,10 +1144,11 @@ void DB::open(const std::string& path, bool no_create_file, const DBOptions& opt
             cfg.read_only = false;
             cfg.skip_validate = !begin_new_session;
             cfg.disable_sync = options.durability == Durability::MemOnly || options.durability == Durability::Unsafe;
+            cfg.clear_file_on_error = options.clear_on_invalid_file;
 
             // only the session initiator is allowed to create the database, all other
             // must assume that it already exists.
-            cfg.no_create = (begin_new_session ? no_create_file : true);
+            cfg.no_create = (begin_new_session ? options.no_create : true);
 
             // if we're opening a MemOnly file that isn't already opened by
             // someone else then it's a file which should have been deleted on
@@ -1499,8 +1500,7 @@ void DB::open(Replication& repl, const std::string& file, const DBOptions& optio
 
     set_replication(&repl);
 
-    bool no_create = false;
-    open(file, no_create, options); // Throws
+    open(file, options); // Throws
 }
 
 class DBLogger : public Logger {
@@ -1532,7 +1532,7 @@ void DB::set_logger(const std::shared_ptr<util::Logger>& logger) noexcept
         m_logger = std::make_shared<DBLogger>(logger, m_log_id);
 }
 
-void DB::open(Replication& repl, const DBOptions options)
+void DB::open(Replication& repl, const DBOptions& options)
 {
     REALM_ASSERT(!is_attached());
     repl.initialize(*this); // Throws
@@ -2808,10 +2808,10 @@ inline DB::DB(Private, const DBOptions& options)
     }
 }
 
-DBRef DB::create(const std::string& file, bool no_create, const DBOptions& options) NO_THREAD_SAFETY_ANALYSIS
+DBRef DB::create(const std::string& file, const DBOptions& options) NO_THREAD_SAFETY_ANALYSIS
 {
     DBRef retval = std::make_shared<DB>(Private(), options);
-    retval->open(file, no_create, options);
+    retval->open(file, options);
     return retval;
 }
 

src/realm/db.hpp

@@ -128,7 +128,7 @@ class DB : public std::enable_shared_from_this<DB> {
     // calling DB::close(), but after that no new association can be established. To reopen the
     // file (or another file), a new DB object is needed. The specified Replication instance, if
     // any, must remain in existence for as long as the DB.
-    static DBRef create(const std::string& file, bool no_create = false, const DBOptions& options = DBOptions());
+    static DBRef create(const std::string& file, const DBOptions& options = DBOptions());
     static DBRef create(Replication& repl, const std::string& file, const DBOptions& options = DBOptions());
     static DBRef create(std::unique_ptr<Replication> repl, const std::string& file,
                         const DBOptions& options = DBOptions());
@@ -534,10 +534,6 @@ class DB : public std::enable_shared_from_this<DB> {
     ///
     /// \param file Filesystem path to a Realm database file.
     ///
-    /// \param no_create If the database file does not already exist, it will be
-    /// created (unless this is set to true.) When multiple threads are involved,
-    /// it is safe to let the first thread, that gets to it, create the file.
-    ///
     /// \param options See DBOptions for details of each option.
     /// Sensible defaults are provided if this parameter is left out.
     ///
@@ -553,13 +549,12 @@ class DB : public std::enable_shared_from_this<DB> {
     /// \throw UnsupportedFileFormatVersion if the file format version or
     /// history schema version is one which this version of Realm does not know
     /// how to migrate from.
-    void open(const std::string& file, bool no_create = false, const DBOptions& options = DBOptions())
-        REQUIRES(!m_mutex);
+    void open(const std::string& file, const DBOptions& options = DBOptions()) REQUIRES(!m_mutex);
     void open(BinaryData, bool take_ownership = true) REQUIRES(!m_mutex);
     void open(Replication&, const std::string& file, const DBOptions& options = DBOptions()) REQUIRES(!m_mutex);
-    void open(Replication& repl, const DBOptions options = DBOptions()) REQUIRES(!m_mutex);
+    void open(Replication& repl, const DBOptions& options = DBOptions()) REQUIRES(!m_mutex);
 
-    void do_open(const std::string& file, bool no_create, const DBOptions& options);
+    void do_open(const std::string& file, const DBOptions& options);
 
     Replication* const* get_repl() const noexcept
     {

src/realm/db_options.hpp

@@ -88,6 +88,9 @@ struct DBOptions {
     /// Disable automatic backup at file format upgrade by setting to false
     bool backup_at_file_format_change = true;
 
+    /// Disable creating new files if the DB to open does not already exist.
+    bool no_create = false;
+
     /// List of versions we can upgrade from
     BackupHandler::VersionList accepted_versions = BackupHandler::accepted_versions_;
 
@@ -99,6 +102,10 @@ struct DBOptions {
     /// a performance impact.
     bool enable_async_writes = false;
 
+    /// If set, opening a file which is not a Realm file or cannot be decrypted
+    /// will clear and reinitialize the file.
+    bool clear_on_invalid_file = false;
+
     /// sys_tmp_dir will be used if the temp_dir is empty when creating DBOptions.
     /// It must be writable and allowed to create pipe/fifo file on it.
     /// set_sys_tmp_dir is not a thread-safe call and it is only supposed to be called once

src/realm/object-store/impl/realm_coordinator.cpp

@@ -479,6 +479,7 @@ bool RealmCoordinator::open_db()
         }
         options.encryption_key = m_config.encryption_key.data();
         options.allow_file_format_upgrade = !m_config.disable_format_upgrade && !schema_mode_reset_file;
+        options.clear_on_invalid_file = m_config.clear_on_invalid_file;
         if (history) {
             options.backup_at_file_format_change = m_config.backup_at_file_format_change;
 #ifdef __EMSCRIPTEN__
@@ -496,7 +497,8 @@ bool RealmCoordinator::open_db()
 #endif
         }
         else {
-            m_db = DB::create(m_config.path, true, options);
+            options.no_create = true;
+            m_db = DB::create(m_config.path, options);
         }
     }
     catch (realm::FileFormatUpgradeRequired const&) {

src/realm/object-store/shared_realm.hpp

@@ -184,6 +184,13 @@ struct RealmConfig {
     // everything can be done deterministically on one thread, and
     // speeds up tests that don't need notifications.
     bool automatic_change_notifications = true;
+
+    // For internal use and should not be exposed by SDKs.
+    //
+    // If the file is invalid or can't be decrypted with the given encryption
+    // key, clear it and reinitialize it as a new file. This is used for the
+    // sync metadata realm which is automatically deleted if it can't be used.
+    bool clear_on_invalid_file = false;
 };
 
 class Realm : public std::enable_shared_from_this<Realm> {