Fix Bm25Stats underflow bug (#26435)

GitOrigin-RevId: ca9793d438b2b5a215cd0288f5f42b5259893a76

Author: emmaling27

crates/database/src/tests/randomized_search_tests.rs

@@ -755,6 +755,26 @@ async fn test_filtering_match_updates(rt: TestRuntime) -> anyhow::Result<()> {
     anyhow::Ok(())
 }
 
+#[convex_macro::test_runtime]
+async fn test_bm25_stats_no_underflow(rt: TestRuntime) -> anyhow::Result<()> {
+    let mut scenario = Scenario::new(rt).await?;
+    scenario
+        .patch(TestKey::C, vec![TestValue::D], TestValue::A)
+        .await?;
+    scenario.execute(TestAction::Backfill).await?;
+    scenario.execute(TestAction::Delete(TestKey::C)).await?;
+    // This query doens't use the filter field, so the BM25 stats will not include
+    // the filter field while the commit statistics will in the memory index from
+    // the delete.
+    scenario
+        .execute(TestAction::QueryAndCheckScores(TestQuery {
+            search: vec![TestValue::D],
+            filter: None,
+        }))
+        .await?;
+    anyhow::Ok(())
+}
+
 // Regression test: We had a bug where we were computing the index of a matching
 // union term incorrectly.
 //

crates/search/src/memory_index/mod.rs

@@ -464,10 +464,19 @@ impl MemorySearchIndex {
                 .checked_add_signed(commit_stats.total_docs_diff as i64)
                 .context("num_documents underflow")?;
             for (field, total_term_diff) in &commit_stats.total_term_diff_by_field {
-                let term_diff = stats.num_terms_by_field.entry(*field).or_insert(0);
-                *term_diff = term_diff
-                    .checked_add_signed(*total_term_diff as i64)
-                    .context("num_terms underflow")?;
+                // It's possible some fields are present in the commit statistics but not the
+                // Bm25Stats because the Bm25Stats are query-specific, and the commit statistics
+                // are not. e.g. a filter field that isn't in the query might not appear in the
+                // Bm25 stats. We only need to update the fields that are in the Bm25Stats.
+                if let Some(term_diff) = stats.num_terms_by_field.get_mut(field) {
+                    *term_diff = term_diff
+                        .checked_add_signed(*total_term_diff as i64)
+                        .context("num_terms underflow")?;
+                } else if field == &Field::from_field_id(SEARCH_FIELD_ID) {
+                    stats
+                        .num_terms_by_field
+                        .insert(*field, (*total_term_diff as i64).try_into()?);
+                }
             }
             for (term, term_id) in &term_ids {
                 let Some(&increment) = commit_stats.term_freq_diffs.get(term_id) else {