Merge pull request #2 from openaleph/feat/tagging

Entity tagging

Author: simonwoerpel

.github/workflows/build-elastic.yml

@@ -23,9 +23,9 @@ jobs:
           images: ghcr.io/openaleph/elasticsearch
           tags: |
             type=ref,event=branch
-            type=semver,pattern={{version}}
             type=sha
             type=raw,value=9
+            type=raw,value=9.1.1
             type=raw,value=latest
 
       - name: Set up Docker Buildx

openaleph_search/index/entities.py

@@ -28,6 +28,7 @@
     "caption",
     "schema",
     "properties",
+    "tags",
     "dataset",
     "collection_id",
     "profile_id",

openaleph_search/index/mapping.py

@@ -107,6 +107,7 @@ class Field:
     GEO_POINT = "geo_point"
     CONTENT = "content"
     TEXT = "text"
+    TAGS = "tags"
 
     NUMERIC = "numeric"
     PROPERTIES = "properties"
@@ -228,6 +229,8 @@ class FieldType:
     # full text
     Field.CONTENT: FieldType.CONTENT,
     Field.TEXT: FieldType.TEXT,
+    # tagging
+    Field.TAGS: FieldType.KEYWORD,
     # processing metadata
     Field.UPDATED_AT: FieldType.DATE,
     Field.CREATED_AT: FieldType.DATE,

openaleph_search/transform/entity.py

@@ -89,6 +89,9 @@ def format_entity(dataset: str, entity: EntityProxy, **kwargs) -> Action | None:
     data[Field.NAME_PARTS] = list(index_name_parts(entity.schema, names))
     data[Field.NAME_PHONETIC] = list(phonetic_names(entity.schema, names))
 
+    # Add tags from EntityProxy.context (they are added from aleph db before indexing)
+    data[Field.TAGS] = ensure_list(entity.context.get("tags"))
+
     # Slight hack: a magic property in followthemoney that gets taken out
     # of the properties and added straight to the index text.
     properties = data.get("properties", {})

tests/test_indexer.py

@@ -1,5 +1,8 @@
+from ftmq.util import make_entity
+
 from openaleph_search.index.admin import clear_index
 from openaleph_search.index.entities import index_bulk, iter_entities
+from openaleph_search.transform.entity import format_entity
 
 
 def test_indexer(entities, cleanup_after):
@@ -12,3 +15,38 @@ def test_indexer(entities, cleanup_after):
     # overwrite
     index_bulk("test_dataset", entities)
     assert len(list(iter_entities())) == 21
+
+
+def test_indexer_with_tags(cleanup_after):
+    # clear
+    clear_index()
+
+    # Create entity with tags in context
+    entity_data = {
+        "id": "test-person-with-tags",
+        "schema": "Person",
+        "properties": {"name": ["Jane Doe"], "birthDate": ["1980-01-01"]},
+    }
+    entity = make_entity(entity_data)
+    entity.context["tags"] = ["politician", "businessman", "controversial"]
+
+    # Verify format_entity includes the tags
+    formatted = format_entity("test_dataset", entity)
+    assert formatted is not None
+    assert "tags" in formatted["_source"]
+    assert formatted["_source"]["tags"] == [
+        "politician",
+        "businessman",
+        "controversial",
+    ]
+
+    index_bulk("test_dataset", [entity])
+
+    # Verify entity was indexed
+    indexed_entities = list(iter_entities())
+    assert len(indexed_entities) == 1
+
+    # Verify tags are in the indexed document
+    indexed_entity = indexed_entities[0]
+    assert "tags" in indexed_entity
+    assert indexed_entity["tags"] == ["politician", "businessman", "controversial"]

tests/test_search.py

@@ -321,3 +321,48 @@ def test_search_sort(cleanup_after):
     result = query.search()
     assert len(result["hits"]["hits"]) == 2
     assert result["hits"]["hits"][0]["_id"] == "event2"
+
+
+def test_search_tag_filter(cleanup_after):
+    # Create entity with tags in context
+    entity_data = {
+        "id": "jane-doe-tagged",
+        "schema": "Person",
+        "properties": {"name": ["Jane Doe"], "birthDate": ["1980-01-01"]},
+    }
+    entity = make_entity(entity_data)
+    entity.context["tags"] = ["politician", "businessman", "controversial"]
+
+    index_bulk("test_tagged", [entity], sync=True)
+
+    # Test search with tag filter
+    query = _create_query("/search?filter:dataset=test_tagged&filter:tags=politician")
+    result = query.search()
+
+    assert result["hits"]["total"]["value"] == 1
+    assert result["hits"]["hits"][0]["_id"] == "jane-doe-tagged"
+
+    # Test search with non-existent tag
+    query = _create_query("/search?filter:dataset=test_tagged&filter:tags=nonexistent")
+    result = query.search()
+
+    assert result["hits"]["total"]["value"] == 0
+
+    # Test search with multiple tag values
+    query = _create_query("/search?filter:dataset=test_tagged&filter:tags=businessman")
+    result = query.search()
+
+    assert result["hits"]["total"]["value"] == 1
+    assert result["hits"]["hits"][0]["_id"] == "jane-doe-tagged"
+
+    # Test tags facet aggregation
+    query = _create_query("/search?filter:dataset=test_tagged&facet=tags")
+    result = query.search()
+
+    assert result["hits"]["total"]["value"] == 1
+    assert "aggregations" in result
+    assert result["aggregations"]["tags.values"]["buckets"] == [
+        {"key": "businessman", "doc_count": 1},
+        {"key": "controversial", "doc_count": 1},
+        {"key": "politician", "doc_count": 1},
+    ]