🔧 (highlighting) Disable FVH per default

Author: simonwoerpel

openaleph_search/settings.py

@@ -80,7 +80,7 @@ class Settings(BaseSettings):
     content_term_vectors: bool = True
 
     # Highlighter configuration
-    highlighter_fvh_enabled: bool = True
+    highlighter_fvh_enabled: bool = False
     highlighter_fragment_size: int = 200
     highlighter_number_of_fragments: int = 3
     highlighter_phrase_limit: int = 64

tests/test_highlighting.py

@@ -25,22 +25,26 @@ def _search_highlight(
 def test_highlighting_phrases(index_entities):
     # this is using the sample entity 242d6724b38425f11df37437c38125b71fb13300
     highlight = _search_highlight('"Mr. Trump proclaimed"')
-    assert "<em>Mr. Trump proclaimed</em>" in highlight
+    # Unified highlighter highlights individual terms rather than full phrase spans
+    assert "<em>Mr</em>" in highlight
+    assert "<em>Trump</em>" in highlight
+    assert "<em>proclaimed</em>" in highlight
 
     highlight = _search_highlight('"former chairman"~2')
     assert "<em>former</em>" in highlight
     assert "<em>chairman</em>" in highlight
 
     highlight = _search_highlight('"paul manafort"')
     assert highlight is not None
-    assert "<em>Paul Manafort</em>" in highlight
+    assert "<em>Paul</em>" in highlight
+    assert "<em>Manafort</em>" in highlight
 
     highlight = _search_highlight("Українська")
     assert highlight is not None
     assert "<em>Українська" in highlight
     highlight = _search_highlight('"日本語"')
     assert highlight is not None
-    assert "<em>本</em><em>語" in highlight
+    assert "<em>日本語" in highlight
 
 
 def test_highlighting_pages(fixture_pages, cleanup_after):
@@ -49,11 +53,14 @@ def test_highlighting_pages(fixture_pages, cleanup_after):
     index_bulk("test_pages_highlight", fixture_pages, sync=True)
 
     # Search in the indexText (-> content) of a Pages (aka Document) entity
+    # Unified highlighter highlights individual terms rather than full phrase spans
     highlight = _search_highlight(
         '"MIT license" "useful information" documentation', schema="Pages"
     )
-    assert "<em>MIT license</em>" in highlight
-    assert "<em>useful information</em>" in highlight
+    assert "<em>MIT</em>" in highlight
+    assert "<em>license</em>" in highlight
+    assert "<em>useful</em>" in highlight
+    assert "<em>information</em>" in highlight
     assert "<em>documentation</em>" in highlight
 
     # Search within its child page entities (used in OpenAleph ui)
@@ -62,24 +69,28 @@ def test_highlighting_pages(fixture_pages, cleanup_after):
         schema="Page",
         parent_id="f61295777cf69f423855655f1614794ce22086d8.b154e50f50c8c8133168767d78bbd1dff067f308",
     )
-    assert "<em>MIT license</em>" in highlight
-    assert "<em>useful information</em>" in highlight
+    assert "<em>MIT</em>" in highlight
+    assert "<em>license</em>" in highlight
+    assert "<em>useful</em>" in highlight
+    assert "<em>information</em>" in highlight
     assert "<em>documentation</em>" in highlight
 
     # Include mentioned names
     highlight = _search_highlight(
         'names:"massachusetts institute of technology" "MIT license"', schema="Pages"
     )
     assert "massachusetts institute of technology" in highlight
-    assert "<em>MIT license</em>" in highlight
+    assert "<em>MIT</em>" in highlight
+    assert "<em>license</em>" in highlight
 
     # Page doesn't contain "names" but still the highlight works for the text phrase
     highlight = _search_highlight(
         'names:"massachusetts institute of technology" "MIT license"',
         schema="Page",
         parent_id="f61295777cf69f423855655f1614794ce22086d8.b154e50f50c8c8133168767d78bbd1dff067f308",
     )
-    assert "<em>MIT license</em>" in highlight
+    assert "<em>MIT</em>" in highlight
+    assert "<em>license</em>" in highlight
 
 
 def test_highlighting_translation_plaintext(cleanup_after):