diff --git a/src/progress.rs b/src/progress.rs index be0f9b1..f05d9ca 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -3,6 +3,7 @@ use steppe::{make_atomic_progress, make_enum_progress}; make_enum_progress! { pub enum HannoyBuild { RetrievingTheItemsIds, + DeletingTheLinks, RetrieveTheUpdatedItems, FetchItemPointers, FetchLinksPointers, diff --git a/src/writer.rs b/src/writer.rs index 2286b5d..95be453 100644 --- a/src/writer.rs +++ b/src/writer.rs @@ -558,7 +558,7 @@ impl Writer { // Remove deleted links from lmdb AFTER build; in DiskANN we use a deleted item's // neighbours when filling in the "gaps" left in the graph from deletions. See // [`HnswBuilder::maybe_patch_old_links`] for more details. - self.delete_links_from_db(&to_delete, wtxn)?; + self.delete_links_from_db(&to_delete, wtxn, options)?; debug!("write the metadata..."); options.progress.update(HannoyBuild::WriteTheMetadata); @@ -585,6 +585,10 @@ impl Writer { } /// Kinda like clear and create, but only for links + /// + /// You must ensure that no items were inserted that are not + /// part of the metadata. Which means that a build must have + /// been performed or you'll see item leaking. fn force_rebuild( &self, wtxn: &mut RwTxn, @@ -601,15 +605,21 @@ impl Writer { "forcing relinking of all items requires the relink_all_items option to be set to true" ); - // 2. delete metadata + // 2. Fetch the list of items from the metadata + let Metadata { items: item_ids, .. } = self + .database + .remap_data_type::() + .get(wtxn, &Key::metadata(self.index))? + .expect("The metadata must be there"); + + // 3. delete metadata self.database.delete(wtxn, &Key::metadata(self.index))?; - // 3. delete version + // 4. delete version self.database.delete(wtxn, &Key::version(self.index))?; - // 4. delete all links - let item_ids = self.item_indices(wtxn, options)?; - self.delete_links_from_db(&item_ids, wtxn)?; + // 5. delete all links + self.delete_links_from_db(&item_ids, wtxn, options)?; // 5. trigger build self.build::(wtxn, rng, options) @@ -682,7 +692,18 @@ impl Writer { // Iterates over links in lmdb and deletes those in `to_delete`. There can be several links // with the same NodeId.item, each differing by their layer - fn delete_links_from_db(&self, to_delete: &RoaringBitmap, wtxn: &mut RwTxn) -> Result<()> { + fn delete_links_from_db

( + &self, + to_delete: &RoaringBitmap, + wtxn: &mut RwTxn, + options: &BuildOption

, + ) -> Result<()> + where + P: steppe::Progress, + { + debug!("started deleting the links..."); + options.progress.update(HannoyBuild::DeletingTheLinks); + let mut cursor = self .database .remap_key_type::()