✨ Store translations separately for highlighting

Author: simonwoerpel

openaleph_search/index/indexes.py

@@ -114,8 +114,10 @@ def make_schema_bucket_mapping(bucket: Bucket) -> dict[str, Any]:
     if bucket == "pages":
         # store full text for highlighting
         mapping["properties"][Field.CONTENT]["store"] = True
+        mapping["properties"][Field.TRANSLATION]["store"] = True
     else:
         mapping["properties"][Field.CONTENT]["store"] = False
+        mapping["properties"][Field.TRANSLATION]["store"] = False
     return mapping
 
 

openaleph_search/index/mapping.py

@@ -16,6 +16,8 @@
 MappingProperty: TypeAlias = dict[str, list[str] | str]
 Mapping: TypeAlias = dict[str, MappingProperty]
 
+PROP_TRANSLATED = "translatedText"
+
 # MAPPING SHORTCUTS #
 DEFAULT_ANALYZER = "default"
 DEFAULT_NORMALIZER = "default"
@@ -95,6 +97,7 @@ class Field:
     CONTENT = "content"
     TEXT = "text"
     TAGS = "tags"
+    TRANSLATION = "translation"
 
     NUMERIC = "numeric"
     PROPERTIES = "properties"
@@ -184,6 +187,7 @@ class FieldType:
             *GROUPS,
             Field.TEXT,
             Field.CONTENT,
+            Field.TRANSLATION,
             Field.NAME,
             Field.NAME_KEYS,
             Field.NAME_PARTS,
@@ -217,6 +221,7 @@ class FieldType:
     # full text
     Field.CONTENT: FieldType.CONTENT,
     Field.TEXT: FieldType.TEXT,
+    Field.TRANSLATION: FieldType.TEXT,
     # tagging
     Field.TAGS: FieldType.KEYWORD,
     # processing metadata
@@ -303,7 +308,9 @@ def make_schema_mapping(schemata: Iterable[SchemaType]) -> Mapping:
             if prop.stub:
                 continue
             merged_props[name]["type"].add(get_index_field_type(prop.type))
-            if prop.type == registry.text:
+            if name == PROP_TRANSLATED:
+                merged_props[name]["copy_to"].add(Field.TRANSLATION)
+            elif prop.type == registry.text:
                 merged_props[name]["copy_to"].add(Field.CONTENT)
             else:
                 merged_props[name]["copy_to"].add(Field.TEXT)

openaleph_search/query/base.py

@@ -375,6 +375,7 @@ def get_highlight(self) -> dict[str, Any]:
             ),
             Field.NAMES: get_highlighter(Field.NAME),
             Field.NAMES: get_highlighter(Field.NAMES),
+            Field.TRANSLATION: get_highlighter(Field.TRANSLATION, query),
         }
         if Field.TEXT not in fields:
             fields[Field.TEXT] = get_highlighter(Field.TEXT, query)

openaleph_search/query/queries.py

@@ -56,6 +56,7 @@ class EntitiesQuery(Query):
         f"{Field.NAME_PARTS}^2",
         Field.CONTENT,
         f"{Field.TEXT}^0.8",
+        f"{Field.TRANSLATION}^0.7",
     ]
     PREFIX_FIELD = Field.NAME_PARTS
     HIGHLIGHT_FIELD = Field.CONTENT

openaleph_search/transform/entity.py

@@ -8,12 +8,12 @@
 from multiprocessing import cpu_count
 from typing import Generator, Iterable
 
-from anystore.functools import weakref_cache
 from anystore.logging import get_logger
 from anystore.util import Took
 from banal import ensure_list
 from followthemoney import EntityProxy, model, registry
 from followthemoney.namespace import Namespace
+from ftmq.util import select_data
 
 from openaleph_search.index.indexer import Action, Actions
 from openaleph_search.index.indexes import entities_write_index, schema_bucket
@@ -53,7 +53,11 @@ def _get_symbols(entity: EntityProxy) -> set[str]:
     return symbols
 
 
-@weakref_cache
+def _get_translations(entity: EntityProxy) -> set[str]:
+    return set(select_data(entity, "__translation__"))
+
+
+@functools.cache
 def _get_namespace(value: str) -> Namespace:
     return Namespace(value)
 
@@ -78,6 +82,7 @@ def format_entity(dataset: str, entity: EntityProxy, **kwargs) -> Action | None:
     dataset = valid_dataset(dataset)
 
     data = entity.to_dict()
+    data["properties"] = data.get("properties", {})
     # deprecated
     collection_id = kwargs.get("collection_id")
     if collection_id is not None:
@@ -109,13 +114,21 @@ def format_entity(dataset: str, entity: EntityProxy, **kwargs) -> Action | None:
 
     # Slight hack: a magic property in followthemoney that gets taken out
     # of the properties and added straight to the index text.
-    properties = data.get("properties", {})
-    text = properties.pop("indexText", [])
+    text = data["properties"].pop("indexText", [])
+
+    # Another hack: Translations are prefixed with "__translation__" in
+    # `Pages.indexText`
+    if entity.schema.name == "Pages":
+        translations = _get_translations(entity)
+        if translations:
+            data[Field.TRANSLATION] = list(translations)
+            text = [t for t in text if not t.startswith("__translation__")]
+
     if text:
         data[Field.CONTENT] = text
 
     # length normalization
-    data[Field.NUM_VALUES] = sum([len(v) for v in properties.values()])
+    data[Field.NUM_VALUES] = sum([len(v) for v in data["properties"].values()])
 
     # integer casting
     numeric = {}

tests/test_highlighting.py

@@ -1,3 +1,5 @@
+from ftmq.util import make_entity
+
 from openaleph_search.index.entities import index_bulk
 from openaleph_search.parse.parser import SearchQueryParser
 from openaleph_search.query.queries import EntitiesQuery
@@ -78,3 +80,92 @@ def test_highlighting_pages(fixture_pages, cleanup_after):
         parent_id="f61295777cf69f423855655f1614794ce22086d8.b154e50f50c8c8133168767d78bbd1dff067f308",
     )
     assert "<em>MIT license</em>" in highlight
+
+
+def test_highlighting_translation_plaintext(cleanup_after):
+    """Test that translation highlights are returned under the 'translation' key,
+    separate from 'content' highlights."""
+    entity = make_entity(
+        {
+            "id": "plaintext-highlight-translation",
+            "schema": "PlainText",
+            "properties": {
+                "fileName": ["report.txt"],
+                "bodyText": ["Original text about financial regulations"],
+                "translatedText": ["Übersetzter Text über Finanzvorschriften"],
+            },
+        }
+    )
+
+    index_bulk("test_highlight_translation", [entity], sync=True)
+
+    args = [
+        ("q", "Finanzvorschriften"),
+        ("highlight", "true"),
+        ("filter:dataset", "test_highlight_translation"),
+    ]
+    query = EntitiesQuery(SearchQueryParser(args, None))
+    result = query.search()
+
+    assert result["hits"]["total"]["value"] == 1
+    hit = result["hits"]["hits"][0]
+    assert "highlight" in hit
+    # Translation highlights should be under the 'translation' key
+    assert "translation" in hit["highlight"]
+    assert any(
+        "<em>Finanzvorschriften</em>" in fragment
+        for fragment in hit["highlight"]["translation"]
+    )
+
+
+def test_highlighting_translation_pages(cleanup_after):
+    """Test that Pages translation highlights are returned under 'translation' key,
+    while original content highlights come under 'content'."""
+    entity = make_entity(
+        {
+            "id": "pages-highlight-translation",
+            "schema": "Pages",
+            "properties": {
+                "fileName": ["bericht.pdf"],
+                "indexText": [
+                    "Original German text about environmental policies",
+                    "__translation__ Translated text about Umweltpolitik",
+                ],
+            },
+        }
+    )
+
+    index_bulk("test_highlight_pages_translation", [entity], sync=True)
+
+    # Search for a term in the translation
+    args = [
+        ("q", "Umweltpolitik"),
+        ("highlight", "true"),
+        ("filter:dataset", "test_highlight_pages_translation"),
+    ]
+    query = EntitiesQuery(SearchQueryParser(args, None))
+    result = query.search()
+
+    assert result["hits"]["total"]["value"] == 1
+    hit = result["hits"]["hits"][0]
+    assert "highlight" in hit
+    assert "translation" in hit["highlight"]
+    assert any(
+        "<em>Umweltpolitik</em>" in fragment
+        for fragment in hit["highlight"]["translation"]
+    )
+
+    # Search for a term in the original content — should highlight under 'content'
+    args = [
+        ("q", "environmental policies"),
+        ("highlight", "true"),
+        ("filter:dataset", "test_highlight_pages_translation"),
+    ]
+    query = EntitiesQuery(SearchQueryParser(args, None))
+    result = query.search()
+
+    assert result["hits"]["total"]["value"] == 1
+    hit = result["hits"]["hits"][0]
+    assert "highlight" in hit
+    assert "content" in hit["highlight"]
+    assert "translation" not in hit["highlight"]

tests/test_indexer.py

@@ -190,6 +190,59 @@ def test_bulk_indexing_mode(cleanup_after):
         assert idx_settings["number_of_replicas"] == str(settings.index_replicas)
 
 
+def test_translation_plaintext():
+    """PlainText: translatedText property is kept in properties; the ES mapping
+    copy_to directive copies it into the `translation` field at index time, so
+    the transform payload should NOT contain a top-level `translation` key."""
+    entity = make_entity(
+        {
+            "id": "plain-text-translated",
+            "schema": "PlainText",
+            "properties": {
+                "fileName": ["document.txt"],
+                "translatedText": ["This is the translated text"],
+            },
+        }
+    )
+    action = format_entity("test_dataset", entity)
+    assert action is not None
+    source = action["_source"]
+    # translatedText stays in properties for ES copy_to to handle
+    assert "translatedText" in source["properties"]
+    assert source["properties"]["translatedText"] == ["This is the translated text"]
+    # No explicit translation field — ES copy_to handles it
+    assert "translation" not in source
+
+
+def test_translation_pages():
+    """Pages: translations are extracted from indexText values prefixed with
+    `__translation__` and placed into the `translation` field explicitly."""
+    entity = make_entity(
+        {
+            "id": "pages-translated",
+            "schema": "Pages",
+            "properties": {
+                "fileName": ["document.pdf"],
+                "indexText": [
+                    "regular text content",
+                    "__translation__ Dies ist der übersetzte Text",
+                    "__translation__ Ceci est le texte traduit",
+                ],
+            },
+        }
+    )
+    action = format_entity("test_dataset", entity)
+    assert action is not None
+    source = action["_source"]
+    assert "translation" in source
+    assert set(source["translation"]) == {
+        "Dies ist der übersetzte Text",
+        "Ceci est le texte traduit",
+    }
+    # indexText is moved to `content`, and translations are stripped out
+    assert "content" in source
+
+
 def test_indexer_namespace(monkeypatch):
     import importlib
 

tests/test_mapping.py

@@ -167,7 +167,12 @@ def test_mapping_spec():
     assert GROUP_MAPPING["emails"]["type"] == "keyword"
     mapping = make_schema_mapping(model.schemata.values())
     assert all(
-        ("text" in f["copy_to"] or "content" in f["copy_to"]) for f in mapping.values()
+        (
+            "text" in f["copy_to"]
+            or "content" in f["copy_to"]
+            or "translation" in f["copy_to"]
+        )
+        for f in mapping.values()
     )
     assert "names" in mapping["name"]["copy_to"]
     assert "dates" in mapping["birthDate"]["copy_to"]
@@ -188,6 +193,12 @@ def test_mapping_spec():
         "name" in person_mapping["name"]["copy_to"]
     ), "Person name should copy to name field"
 
+    # translatedText should copy to translation field, not to content/text
+    plaintext_mapping = make_schema_mapping(["PlainText"])
+    assert "translation" in plaintext_mapping["translatedText"]["copy_to"]
+    assert "content" not in plaintext_mapping["translatedText"]["copy_to"]
+    assert "text" not in plaintext_mapping["translatedText"]["copy_to"]
+
     full_mapping = make_mapping(mapping)
     assert "date" in full_mapping["properties"]["numeric"]["properties"]
     assert "dates" in full_mapping["properties"]["numeric"]["properties"]
@@ -197,14 +208,11 @@ def test_mapping_spec():
 
 
 def test_mapping_schema_bucket():
-    # full text is stored only for Pages entities
+    # full text and translation are stored only for Pages entities (for highlighting)
     mapping = make_schema_bucket_mapping("pages")
     assert mapping["properties"]["content"]["store"] is True
-    mapping = make_schema_bucket_mapping("page")
-    assert mapping["properties"]["content"]["store"] is False
-    mapping = make_schema_bucket_mapping("documents")
-    assert mapping["properties"]["content"]["store"] is False
-    mapping = make_schema_bucket_mapping("intervals")
-    assert mapping["properties"]["content"]["store"] is False
-    mapping = make_schema_bucket_mapping("things")
-    assert mapping["properties"]["content"]["store"] is False
+    assert mapping["properties"]["translation"]["store"] is True
+    for bucket in ("page", "documents", "intervals", "things"):
+        mapping = make_schema_bucket_mapping(bucket)
+        assert mapping["properties"]["content"]["store"] is False
+        assert mapping["properties"]["translation"]["store"] is False

tests/test_pages.py

@@ -38,3 +38,50 @@ def test_pages(fixture_pages, cleanup_after):
     assert len(result["hits"]["hits"]) == 1
     assert result["hits"]["hits"][0]["_source"]["schema"] == "Pages"
     assert "<em>MIT</em>" in result["hits"]["hits"][0]["highlight"]["content"][0]
+
+
+def test_pages_translation(cleanup_after):
+    """Test that Pages entities with __translation__-prefixed indexText values
+    are searchable via the translation field."""
+    from ftmq.util import make_entity
+
+    entity = make_entity(
+        {
+            "id": "pages-with-translation",
+            "schema": "Pages",
+            "properties": {
+                "fileName": ["bericht.pdf"],
+                "indexText": [
+                    "Dies ist der Originalbericht auf Deutsch",
+                    "__translation__ This is the translated report in English",
+                    "__translation__ Ceci est le rapport traduit en français",
+                ],
+            },
+        }
+    )
+
+    index_bulk("test_pages_translation", [entity], sync=True)
+
+    # Search for original German text — should match via content field
+    query = _create_query(
+        "/search?q=Originalbericht&filter:dataset=test_pages_translation"
+    )
+    result = query.search()
+    assert result["hits"]["total"]["value"] == 1
+    assert result["hits"]["hits"][0]["_id"] == "pages-with-translation"
+
+    # Search for English translation — should match via translation field
+    query = _create_query(
+        "/search?q=translated report English&filter:dataset=test_pages_translation"
+    )
+    result = query.search()
+    assert result["hits"]["total"]["value"] == 1
+    assert result["hits"]["hits"][0]["_id"] == "pages-with-translation"
+
+    # Search for French translation
+    query = _create_query(
+        "/search?q=rapport traduit français&filter:dataset=test_pages_translation"
+    )
+    result = query.search()
+    assert result["hits"]["total"]["value"] == 1
+    assert result["hits"]["hits"][0]["_id"] == "pages-with-translation"