🔧 Make MLT tweaks configurable

Author: simonwoerpel

openaleph_search/parse/parser.py

@@ -298,19 +298,34 @@ def get_facet_significant_text_shard_size(self) -> int:
 
     def get_mlt_min_doc_freq(self) -> int:
         """Minimum document frequency for more_like_this query terms."""
-        return self.getint("mlt_min_doc_freq", 1) or 1
+        return self.getint("mlt_min_doc_freq", self.settings.mlt_min_doc_freq) or 1
 
     def get_mlt_minimum_should_match(self) -> str:
         """Minimum should match percentage for more_like_this query."""
-        return self.get("mlt_minimum_should_match", "10%") or "10%"
+        return (
+            self.get("mlt_minimum_should_match", self.settings.mlt_minimum_should_match)
+            or "10%"
+        )
 
     def get_mlt_min_term_freq(self) -> int:
         """Minimum term frequency for more_like_this query terms."""
-        return self.getint("mlt_min_term_freq", 1) or 1
+        return self.getint("mlt_min_term_freq", self.settings.mlt_min_term_freq) or 1
 
     def get_mlt_max_query_terms(self) -> int:
         """Maximum number of query terms for more_like_this query."""
-        return self.getint("mlt_max_query_terms", 200) or 200
+        return (
+            self.getint("mlt_max_query_terms", self.settings.mlt_max_query_terms) or 200
+        )
+
+    def get_mlt_min_word_length(self) -> int:
+        """Minimum word length for more_like_this query terms."""
+        return (
+            self.getint("mlt_min_word_length", self.settings.mlt_min_word_length) or 5
+        )
+
+    def get_mlt_max_doc_freq(self) -> int:
+        """Maximum document frequency for more_like_this query terms."""
+        return self.getint("mlt_max_doc_freq", self.settings.mlt_max_doc_freq) or 500
 
     def to_dict(self) -> dict[str, Any]:
         parser = super().to_dict()

openaleph_search/query/more_like_this.py

@@ -4,8 +4,10 @@
 
 from openaleph_search.index.mapping import Field
 from openaleph_search.query.util import BoolQuery, bool_query, none_query
+from openaleph_search.settings import Settings
 
 log = logging.getLogger(__name__)
+settings = Settings()
 
 
 def more_like_this_query(
@@ -38,17 +40,21 @@ def more_like_this_query(
     elif datasets:
         query["bool"]["filter"].append({"terms": {"dataset": datasets}})
 
-    # Get configurable parameters from parser, with sensible defaults
-    min_doc_freq = 1
-    minimum_should_match = "10%"
-    min_term_freq = 1
-    max_query_terms = 200
+    # Get configurable parameters from parser, falling back to settings defaults
+    min_doc_freq = settings.mlt_min_doc_freq
+    minimum_should_match = settings.mlt_minimum_should_match
+    min_term_freq = settings.mlt_min_term_freq
+    max_query_terms = settings.mlt_max_query_terms
+    min_word_length = settings.mlt_min_word_length
+    max_doc_freq = settings.mlt_max_doc_freq
 
     if parser is not None:
         min_doc_freq = parser.get_mlt_min_doc_freq()
         minimum_should_match = parser.get_mlt_minimum_should_match()
         min_term_freq = parser.get_mlt_min_term_freq()
         max_query_terms = parser.get_mlt_max_query_terms()
+        min_word_length = parser.get_mlt_min_word_length()
+        max_doc_freq = parser.get_mlt_max_doc_freq()
 
     # Build the more_like_this query using document ID
     mlt_query = {
@@ -59,9 +65,8 @@ def more_like_this_query(
             "max_query_terms": max_query_terms,
             "min_doc_freq": min_doc_freq,
             "minimum_should_match": minimum_should_match,
-            # min_word_length filters out short stopwords (of, the, in, ...)
-            "min_word_length": 5,
-            "max_doc_freq": 500,  # filter out very common terms
+            "min_word_length": min_word_length,
+            "max_doc_freq": max_doc_freq,
             "boost_terms": 1,
         }
     }

openaleph_search/settings.py

@@ -82,5 +82,13 @@ class Settings(BaseSettings):
     highlighter_no_match_size: int = 300
     highlighter_max_analyzed_offset: int = 999999
 
+    # More Like This defaults
+    mlt_min_doc_freq: int = 1
+    mlt_min_term_freq: int = 1
+    mlt_max_query_terms: int = 200
+    mlt_minimum_should_match: str = "10%"
+    mlt_min_word_length: int = 5
+    mlt_max_doc_freq: int = 500
+
     # search control
     allow_leading_wildcard: bool = False

pyproject.toml

@@ -65,3 +65,6 @@ build-backend = "poetry.core.masonry.api"
 [tool.pytest_env]
 DEBUG = 1
 OPENALEPH_SEARCH_INDEX_NAMESPACE_IDS = 0
+OPENALEPH_SEARCH_MLT_MIN_WORD_LENGTH = 3
+OPENALEPH_SEARCH_MLT_MIN_DOC_FREQ = 1
+OPENALEPH_SEARCH_MLT_MIN_TERM_FREQ = 1

tests/test_more_like_this.py

@@ -119,7 +119,16 @@ def _create_mlt_query(
 
 @pytest.fixture(scope="function")
 def index_test_documents(cleanup_after):
-    """Index test documents for more_like_this testing"""
+    """Index test documents for more_like_this testing.
+
+    Recreates indexes from scratch to eliminate stale segment metadata from
+    previous tests that can corrupt MLT term statistics (deleted docs remain
+    in Lucene segments and skew IDF calculations).
+    """
+    from openaleph_search.index.admin import delete_index, upgrade_search
+
+    delete_index()
+    upgrade_search()
     entities = [make_entity(doc) for doc in TEST_DOCUMENTS]
     index_bulk("test_mlt", entities, sync=True)
     return entities
@@ -148,11 +157,16 @@ def test_more_like_this_query_function():
     assert "like" in mlt_query
     assert mlt_query["like"] == [{"_id": "doc1"}]
 
-    # Check default parameters (hardcoded in more_like_this.py when no parser)
-    assert mlt_query["min_doc_freq"] == 1
-    assert mlt_query["minimum_should_match"] == "10%"
-    assert mlt_query["min_term_freq"] == 1
-    assert mlt_query["max_query_terms"] == 200
+    # Check default parameters (from settings, no parser)
+    from openaleph_search.settings import Settings
+
+    s = Settings()
+    assert mlt_query["min_doc_freq"] == s.mlt_min_doc_freq
+    assert mlt_query["minimum_should_match"] == s.mlt_minimum_should_match
+    assert mlt_query["min_term_freq"] == s.mlt_min_term_freq
+    assert mlt_query["max_query_terms"] == s.mlt_max_query_terms
+    assert mlt_query["min_word_length"] == s.mlt_min_word_length
+    assert mlt_query["max_doc_freq"] == s.mlt_max_doc_freq
 
     # Test with custom parser
     parser = SearchQueryParser(
@@ -367,10 +381,12 @@ def test_more_like_this_configurable_parameters():
             break
 
     assert mlt_query_default is not None
-    assert mlt_query_default["min_doc_freq"] == 1  # parser default
-    assert mlt_query_default["minimum_should_match"] == "10%"  # parser default
-    assert mlt_query_default["min_term_freq"] == 1  # parser default
-    assert mlt_query_default["max_query_terms"] == 200  # parser default
+    assert mlt_query_default["min_doc_freq"] == 1  # settings default
+    assert mlt_query_default["minimum_should_match"] == "10%"  # settings default
+    assert mlt_query_default["min_term_freq"] == 1  # settings default
+    assert mlt_query_default["max_query_terms"] == 200  # settings default
+    assert mlt_query_default["min_word_length"] == 3  # settings test override
+    assert mlt_query_default["max_doc_freq"] == 500  # settings default
 
 
 def test_more_like_this_bucket_filtering():