feat(core): implement lazy sync for MetadataStore cache

- Remove eager SyncMetadataStoreFromConfigs on notebook open

- Add vxcore_notebook_rebuild_cache API for explicit cache rebuild

- Implement lazy sync in GetFolderConfig - folders synced on access

- Add SyncFolderToStore and GetParentFolderId helpers

- Add test for rebuild cache functionality

Author: tamlok

include/vxcore/vxcore.h

@@ -52,6 +52,12 @@ VXCORE_API VxCoreError vxcore_notebook_update_config(VxCoreContextHandle context
                                                      const char *notebook_id,
                                                      const char *config_json);
 
+// Rebuilds the metadata cache for the notebook from ground truth (vx.json files).
+// Use this when the cache seems out of sync or corrupted.
+// The cache uses lazy sync by default - this forces a full rebuild.
+VXCORE_API VxCoreError vxcore_notebook_rebuild_cache(VxCoreContextHandle context,
+                                                     const char *notebook_id);
+
 VXCORE_API VxCoreError vxcore_folder_create(VxCoreContextHandle context, const char *notebook_id,
                                             const char *parent_path, const char *folder_name,
                                             char **out_folder_id);

src/api/vxcore_notebook_api.cpp

@@ -173,3 +173,29 @@ VXCORE_API VxCoreError vxcore_notebook_update_config(VxCoreContextHandle context
     return VXCORE_ERR_UNKNOWN;
   }
 }
+
+VXCORE_API VxCoreError vxcore_notebook_rebuild_cache(VxCoreContextHandle context,
+                                                     const char *notebook_id) {
+  if (!context || !notebook_id) {
+    return VXCORE_ERR_NULL_POINTER;
+  }
+
+  auto *ctx = reinterpret_cast<vxcore::VxCoreContext *>(context);
+
+  try {
+    auto *notebook = ctx->notebook_manager->GetNotebook(notebook_id);
+    if (!notebook) {
+      ctx->last_error = "Notebook not found";
+      return VXCORE_ERR_NOT_FOUND;
+    }
+
+    VxCoreError err = notebook->RebuildCache();
+    if (err != VXCORE_OK) {
+      ctx->last_error = "Failed to rebuild notebook cache";
+    }
+    return err;
+  } catch (...) {
+    ctx->last_error = "Unknown error rebuilding notebook cache";
+    return VXCORE_ERR_UNKNOWN;
+  }
+}

src/core/bundled_folder_manager.cpp

@@ -204,6 +204,10 @@ VxCoreError BundledFolderManager::GetFolderConfig(const std::string &folder_path
     return error;
   }
 
+  // Lazy sync: populate MetadataStore with this folder's data
+  std::string parent_id = GetParentFolderId(folder_path);
+  SyncFolderToStore(folder_path, *config, parent_id);
+
   *out_config = config.get();
   config_cache_[folder_path] = std::move(config);
   return VXCORE_OK;
@@ -1391,4 +1395,69 @@ VxCoreError BundledFolderManager::SyncMetadataStoreFromConfigs() {
   }
 }
 
+std::string BundledFolderManager::GetParentFolderId(const std::string &folder_path) {
+  if (folder_path.empty() || folder_path == ".") {
+    return "";  // Root folder has no parent
+  }
+
+  const auto [parent_path, folder_name] = SplitPath(folder_path);
+  (void)folder_name;  // Unused
+
+  FolderConfig *parent_config = GetCachedConfig(parent_path);
+  if (parent_config) {
+    return parent_config->id;
+  }
+
+  // Parent not in cache - try to load it (which will also trigger lazy sync for parent)
+  VxCoreError error = GetFolderConfig(parent_path, &parent_config);
+  if (error == VXCORE_OK && parent_config) {
+    return parent_config->id;
+  }
+
+  return "";  // Parent not found
+}
+
+void BundledFolderManager::SyncFolderToStore(const std::string &folder_path,
+                                             const FolderConfig &config,
+                                             const std::string &parent_folder_id) {
+  auto *store = notebook_->GetMetadataStore();
+  if (!store) {
+    return;  // No store available
+  }
+
+  // Check if folder already exists in store
+  auto existing_folder = store->GetFolder(config.id);
+  if (existing_folder.has_value()) {
+    // Folder exists - check if we need to update sync state
+    // For now, we assume the store is up-to-date if folder exists
+    // (write-through cache should keep it in sync)
+    VXCORE_LOG_DEBUG("SyncFolderToStore: Folder already in store: id=%s, path=%s",
+                     config.id.c_str(), folder_path.c_str());
+    return;
+  }
+
+  VXCORE_LOG_INFO("SyncFolderToStore: Adding folder to store: id=%s, path=%s", config.id.c_str(),
+                  folder_path.c_str());
+
+  // Create folder record in store
+  StoreFolderRecord folder_record = ToStoreFolderRecord(config, parent_folder_id);
+  if (!store->CreateFolder(folder_record)) {
+    VXCORE_LOG_WARN("SyncFolderToStore: Failed to create folder in store: id=%s",
+                    config.id.c_str());
+    return;
+  }
+
+  // Create file records in store
+  for (const auto &file : config.files) {
+    StoreFileRecord file_record = ToStoreFileRecord(file, config.id);
+    if (!store->CreateFile(file_record)) {
+      VXCORE_LOG_WARN("SyncFolderToStore: Failed to create file in store: id=%s", file.id.c_str());
+      // Continue anyway - best effort
+    }
+  }
+
+  VXCORE_LOG_DEBUG("SyncFolderToStore: Synced folder with %zu files: path=%s", config.files.size(),
+                   folder_path.c_str());
+}
+
 }  // namespace vxcore

src/core/bundled_folder_manager.h

@@ -100,6 +100,14 @@ class BundledFolderManager : public FolderManager {
 
   FileRecord *FindFileRecord(FolderConfig &config, const std::string &file_name);
 
+  // Sync a single folder's data to MetadataStore after loading from vx.json
+  // This is the lazy sync implementation - called when a folder is accessed
+  void SyncFolderToStore(const std::string &folder_path, const FolderConfig &config,
+                         const std::string &parent_folder_id);
+
+  // Get the parent folder's ID (UUID) for a given folder path
+  std::string GetParentFolderId(const std::string &folder_path);
+
   std::map<std::string, std::unique_ptr<FolderConfig>> config_cache_;
 };
 

src/core/bundled_notebook.cpp

@@ -104,16 +104,9 @@ VxCoreError BundledNotebook::Open(const std::string &local_data_folder,
     return err;
   }
 
-  // Sync MetadataStore from config files (vx.json)
-  // This rebuilds the cache from ground truth on every open
-  auto *bundled_folder_manager =
-      static_cast<BundledFolderManager *>(notebook->folder_manager_.get());
-  err = bundled_folder_manager->SyncMetadataStoreFromConfigs();
-  if (err != VXCORE_OK) {
-    VXCORE_LOG_ERROR("Failed to sync MetadataStore from configs: root=%s, error=%d",
-                     root_folder.c_str(), err);
-    return err;
-  }
+  // Note: We do NOT sync MetadataStore from config files here.
+  // The cache uses lazy sync - data is loaded on demand when accessed.
+  // Users can call RebuildCache() if they need a full refresh.
 
   out_notebook = std::move(notebook);
   return VXCORE_OK;
@@ -170,4 +163,13 @@ VxCoreError BundledNotebook::UpdateConfig(const NotebookConfig &config) {
   }
 }
 
+VxCoreError BundledNotebook::RebuildCache() {
+  auto *bundled_folder_manager = dynamic_cast<BundledFolderManager *>(folder_manager_.get());
+  if (!bundled_folder_manager) {
+    VXCORE_LOG_ERROR("RebuildCache: folder_manager is not BundledFolderManager");
+    return VXCORE_ERR_INVALID_STATE;
+  }
+  return bundled_folder_manager->SyncMetadataStoreFromConfigs();
+}
+
 }  // namespace vxcore

src/core/bundled_notebook.h

@@ -17,6 +17,7 @@ class BundledNotebook : public Notebook {
                           std::unique_ptr<Notebook> &out_notebook);
 
   VxCoreError UpdateConfig(const NotebookConfig &config) override;
+  VxCoreError RebuildCache() override;
 
  private:
   BundledNotebook(const std::string &local_data_folder, const std::string &root_folder);

src/core/notebook.h

@@ -78,6 +78,10 @@ class Notebook {
   VxCoreError GetTags(std::string &out_tags_json) const;
   TagNode *FindTag(const std::string &tag_name);
 
+  // Rebuild the metadata cache from ground truth (config files).
+  // Returns VXCORE_OK on success.
+  virtual VxCoreError RebuildCache() = 0;
+
   // Clean and get path related to notebook root folder.
   // Returns null string if |path| is not under notebook root folder.
   std::string GetCleanRelativePath(const std::string &path) const;

src/core/raw_notebook.cpp

@@ -133,4 +133,10 @@ VxCoreError RawNotebook::UpdateConfig(const NotebookConfig &config) {
   }
 }
 
+VxCoreError RawNotebook::RebuildCache() {
+  // Raw notebooks don't have vx.json config files to sync from.
+  // Metadata is stored only in the database.
+  return VXCORE_OK;
+}
+
 }  // namespace vxcore

src/core/raw_notebook.h

@@ -15,6 +15,7 @@ class RawNotebook : public Notebook {
                           const std::string &id, std::unique_ptr<Notebook> &out_notebook);
 
   VxCoreError UpdateConfig(const NotebookConfig &config) override;
+  VxCoreError RebuildCache() override;
 
  private:
   RawNotebook(const std::string &local_data_folder, const std::string &root_folder);

tests/test_folder.cpp

@@ -1285,7 +1285,9 @@ int test_folder_create_path_trailing_slash() {
 // MetadataStore Sync Integration Tests
 // ============================================================================
 
-// Test that metadata store syncs correctly when notebook is reopened
+// Test that metadata store syncs correctly when notebook is reopened.
+// Note: With lazy sync architecture, data is synced to MetadataStore on-demand
+// when folders/files are accessed, not eagerly on notebook open.
 int test_metadata_store_sync_on_reopen() {
   std::cout << "  Running test_metadata_store_sync_on_reopen..." << std::endl;
   cleanup_test_dir("test_ms_sync_reopen_nb");
@@ -1340,7 +1342,7 @@ int test_metadata_store_sync_on_reopen() {
   vxcore_string_free(notebook_id);
   notebook_id = nullptr;
 
-  // Reopen notebook - this should trigger SyncMetadataStoreFromConfigs
+  // Reopen notebook - lazy sync will populate MetadataStore when folders are accessed
   err = vxcore_notebook_open(ctx, "test_ms_sync_reopen_nb", &notebook_id);
   ASSERT_EQ(err, VXCORE_OK);
   ASSERT_NOT_NULL(notebook_id);