[ENH] Update sparse vector similarity metric (#5406)

## Description of changes

_Summarize the changes made by this PR._

- Improvements & Bug fixes
  - Change sparse vector metric from inner product to one minus inner product. This is consistent with our dense vector similarity metric.
  - Updates `Rank` operator so that the results are returned in increasing score. Smaller score means higher similarity.
  - Removes `SparseKnnMerge` implementation as it is unnecessary now
- New functionality
  - N/A

## Test plan

_How are these changes tested?_

- [ ] Tests pass locally with `pytest` for python, `yarn test` for js, `cargo test` for rust

## Migration plan

_Are there any migrations, or any forwards/backwards compatibility changes needed in order to make sure this change deploys reliably?_

## Observability plan

_What is the plan to instrument and monitor this change?_

## Documentation Changes

_Are all docstrings for user-facing APIs updated if required? Do we need to make documentation changes in the [docs section](https://github.com/chroma-core/chroma/tree/main/docs/docs.trychroma.com)?_

Author: Sicheng-Pan

rust/worker/benches/spann.rs

@@ -212,7 +212,7 @@ fn calculate_recall<'a>(
             }
             // Now merge.
             let knn_input = KnnMergeInput {
-                batch_distances: merge_list,
+                batch_measures: merge_list,
             };
             let knn_operator = Merge { k: k as u32 };
             let knn_output = knn_operator
@@ -245,7 +245,7 @@ fn calculate_recall<'a>(
                 .expect("Error running operator");
             let mut recall = 0;
             for bf_record in bf_output.records.iter() {
-                for spann_record in knn_output.distances.iter() {
+                for spann_record in knn_output.measures.iter() {
                     if bf_record.offset_id == spann_record.offset_id {
                         recall += 1;
                     }

rust/worker/src/execution/operators/knn_merge.rs

@@ -21,12 +21,12 @@ use thiserror::Error;
 
 #[derive(Debug)]
 pub struct KnnMergeInput {
-    pub batch_distances: Vec<Vec<RecordMeasure>>,
+    pub batch_measures: Vec<Vec<RecordMeasure>>,
 }
 
 #[derive(Debug, Default)]
 pub struct KnnMergeOutput {
-    pub distances: Vec<RecordMeasure>,
+    pub measures: Vec<RecordMeasure>,
 }
 
 #[derive(Error, Debug)]
@@ -47,12 +47,12 @@ impl Operator<KnnMergeInput, KnnMergeOutput> for Merge {
         // Reversing because similarity is in ascending order,
         // while Merge takes element in descending order
         let reversed_distances = input
-            .batch_distances
+            .batch_measures
             .iter()
             .map(|batch| batch.iter().map(|m| Reverse(m.clone())).collect())
             .collect();
         Ok(KnnMergeOutput {
-            distances: self
+            measures: self
                 .merge(reversed_distances)
                 .into_iter()
                 .map(|Reverse(distance)| distance)
@@ -74,7 +74,7 @@ mod tests {
     /// - Second: 1, 3, ..., 99
     fn setup_knn_merge_input() -> KnnMergeInput {
         KnnMergeInput {
-            batch_distances: vec![
+            batch_measures: vec![
                 (1..=100)
                     .filter_map(|offset_id| {
                         (offset_id % 3 == 1).then_some(RecordMeasure {
@@ -116,7 +116,7 @@ mod tests {
 
         assert_eq!(
             knn_merge_output
-                .distances
+                .measures
                 .iter()
                 .map(|record| record.offset_id)
                 .collect::<Vec<_>>(),

rust/worker/src/execution/operators/mod.rs

@@ -25,5 +25,4 @@ pub mod repair_log_offsets;
 pub mod select;
 pub mod source_record_segment;
 pub mod sparse_index_knn;
-pub mod sparse_knn_merge;
 pub mod sparse_log_knn;

rust/worker/src/execution/operators/rank.rs

@@ -201,7 +201,6 @@ impl Operator<RankInput, RankOutput> for Rank {
             .map(|(offset_id, measure)| RecordMeasure { offset_id, measure })
             .collect::<Vec<_>>();
         ranks.sort_unstable();
-        ranks.reverse();
         Ok(RankOutput { ranks })
     }
 }
@@ -246,8 +245,9 @@ mod tests {
 
         let output = rank.run(&input).await.expect("Rank should succeed");
         assert_eq!(output.ranks.len(), 3);
-        assert_eq!(output.ranks[0].offset_id, 1);
-        assert_eq!(output.ranks[0].measure, 0.9);
+        // After removing .reverse(), results are in ascending order by measure
+        assert_eq!(output.ranks[0].offset_id, 3);
+        assert_eq!(output.ranks[0].measure, 0.5);
     }
 
     #[tokio::test]
@@ -310,7 +310,11 @@ mod tests {
         };
 
         let output = rank.run(&input).await.expect("Rank should succeed");
-        // Record 1 appears in both: 0.8 + 0.4 = 1.2
+        // Summation results:
+        // Only Record 1 appears in both lists: 0.8 + 0.4 = 1.2
+        // Records 2 and 3 are filtered out since they don't appear in both lists
+        // and both Knn operations have default: None
+        assert_eq!(output.ranks.len(), 1);
         assert_eq!(output.ranks[0].offset_id, 1);
         assert_eq!(output.ranks[0].measure, 1.2);
 
@@ -329,8 +333,10 @@ mod tests {
         let input = RankInput { knn_results };
 
         let output = rank.run(&input).await.expect("Rank should succeed");
-        assert_eq!(output.ranks[0].offset_id, 1);
-        assert_eq!(output.ranks[0].measure, 0.4); // 0.8 * 0.5
+        // Results are in ascending order, so the record with the lowest measure comes first
+        // After multiplication by 0.5: record 1 = 0.8 * 0.5 = 0.4, record 2 = 0.6 * 0.5 = 0.3
+        assert_eq!(output.ranks[0].offset_id, 2);
+        assert_eq!(output.ranks[0].measure, 0.3); // 0.6 * 0.5
     }
 
     #[tokio::test]
@@ -367,10 +373,11 @@ mod tests {
         };
 
         let output = rank.run(&input).await.expect("Rank should succeed");
-        assert_eq!(output.ranks[0].offset_id, 1);
-        assert_eq!(output.ranks[0].measure, 0.8); // max(0.8, 0.5) = 0.8
-        assert_eq!(output.ranks[1].offset_id, 2);
-        assert_eq!(output.ranks[1].measure, 0.5); // max(0.3, 0.5) = 0.5
+        // Results are in ascending order
+        assert_eq!(output.ranks[0].offset_id, 2);
+        assert_eq!(output.ranks[0].measure, 0.5); // max(0.3, 0.5) = 0.5
+        assert_eq!(output.ranks[1].offset_id, 1);
+        assert_eq!(output.ranks[1].measure, 0.8); // max(0.8, 0.5) = 0.8
 
         // Test min
         let rank = Rank::Minimum(vec![
@@ -386,9 +393,10 @@ mod tests {
         let input = RankInput { knn_results };
 
         let output = rank.run(&input).await.expect("Rank should succeed");
-        assert_eq!(output.ranks[0].offset_id, 1);
-        assert_eq!(output.ranks[0].measure, 0.5); // min(0.8, 0.5) = 0.5
-        assert_eq!(output.ranks[1].offset_id, 2);
-        assert_eq!(output.ranks[1].measure, 0.3); // min(0.3, 0.5) = 0.3
+        // Results are in ascending order
+        assert_eq!(output.ranks[0].offset_id, 2);
+        assert_eq!(output.ranks[0].measure, 0.3); // min(0.3, 0.5) = 0.3
+        assert_eq!(output.ranks[1].offset_id, 1);
+        assert_eq!(output.ranks[1].measure, 0.5); // min(0.8, 0.5) = 0.5
     }
 }

rust/worker/src/execution/operators/sparse_index_knn.rs

@@ -68,7 +68,8 @@ impl Operator<SparseIndexKnnInput, SparseIndexKnnOutput> for SparseIndexKnn {
                 .into_iter()
                 .map(|score| RecordMeasure {
                     offset_id: score.offset,
-                    measure: score.score,
+                    // NOTE: We use `1 - query · document` as similarity metrics
+                    measure: 1.0 - score.score,
                 })
                 .collect(),
         })