Adding fixes to allow duplicates search

Author: GavinMendelGleason

parallel-hnsw/src/lib.rs

@@ -1016,6 +1016,7 @@ impl<C: Comparator + 'static> Hnsw<C> {
                     search_parameters.number_of_candidates,
                     search_parameters,
                 );
+
                 last = pq.last().expect("should have at least retrieved self").1;
                 if last < threshold && pq.len() == pq.capacity() {
                     pq.resize_capacity(pq.capacity() * 2);

vectorlink/src/batch.rs

@@ -27,8 +27,8 @@ use urlencoding::encode;
 use crate::{
     comparator::{
         ArrayCentroidComparator, Centroid16Comparator, Centroid16Comparator1024,
-        Centroid8Comparator, Disk1024Comparator, DiskOpenAIComparator, OpenAIComparator,
-        Quantized16Comparator, Quantized16Comparator1024, Quantized8Comparator,
+        Centroid8Comparator, Disk1024Comparator, DiskOpenAIComparator, Memory1024Comparator,
+        OpenAIComparator, Quantized16Comparator, Quantized16Comparator1024, Quantized8Comparator,
     },
     configuration::HnswConfiguration,
     domain::Domain,
@@ -505,7 +505,7 @@ fn perform_indexing(
                 );
                 HnswConfiguration::Quantized1024By16(model, quantized_hnsw)
             } else {
-                let comparator = OpenAIComparator::new(
+                let comparator = Memory1024Comparator::new(
                     domain_obj.name().to_owned(),
                     Arc::new(domain_obj.all_vecs()?),
                 );
@@ -516,7 +516,7 @@ fn perform_indexing(
                     BuildParameters::default(),
                     &mut SimpleProgressMonitor::default(),
                 );
-                HnswConfiguration::UnquantizedOpenAi(model, hnsw)
+                HnswConfiguration::Unquantized1024(model, hnsw)
             };
             eprintln!("done generating hnsw");
             keepalive!(progress, hnsw.serialize(&staging_file))?;

vectorlink/src/comparator.rs

@@ -380,7 +380,7 @@ impl Serializable for Memory1024Comparator {
         let mut contents = String::new();
         comparator_file.read_to_string(&mut contents)?;
         let ComparatorMeta { domain_name, .. } = serde_json::from_str(&contents)?;
-        let domain = store.get_domain_sized(&domain_name, EMBEDDING_BYTE_LENGTH)?;
+        let domain = store.get_domain_sized(&domain_name, EMBEDDING_BYTE_LENGTH_1024)?;
         Ok(Memory1024Comparator {
             domain_name,
             range: Arc::new(domain.all_vecs()?),

vectorlink/src/main.rs

@@ -185,7 +185,7 @@ enum Commands {
         domain: String,
         #[arg(short, long)]
         directory: String,
-        #[arg(short, long)]
+        #[arg(long)]
         take: Option<usize>,
         #[arg(short, long, default_value_t = 10000)]
         size: usize,