exclude obsolete entities from text tagger (#1196)

Author: jamesamcl

apitester4/src/main/java/uk/ac/ebi/ols/apitester/Ols4ApiTester.java

@@ -149,6 +149,14 @@ public boolean testTagText() throws IOException {
 		JsonElement tagSourceFiltered = post(url + "/api/v2/tag_text?source=test-curations", "{\"text\": \"genomic data sharing for clinical trial data\"}");
 		write(outDir + "/v2/tag_text_source_filtered.json", tagSourceFiltered);
 
+		// Test includeObsoleteEntities=true (should include obsolete entity with is_obsolete flag)
+		JsonElement tagWithObsolete = post(url + "/api/v2/tag_text?includeObsoleteEntities=true", "{\"text\": \"testing the obsolete xmpl entity label\"}");
+		write(outDir + "/v2/tag_text_with_obsolete.json", tagWithObsolete);
+
+		// Test includeObsoleteEntities=false (default; should exclude obsolete entity)
+		JsonElement tagNoObsolete = post(url + "/api/v2/tag_text?includeObsoleteEntities=false", "{\"text\": \"testing the obsolete xmpl entity label\"}");
+		write(outDir + "/v2/tag_text_no_obsolete.json", tagNoObsolete);
+
 		return true;
 	}
 

backend/src/main/java/uk/ac/ebi/spot/ols/controller/api/v2/V2TextTaggerController.java

@@ -41,7 +41,8 @@ public HttpEntity<Map<String, Object>> tagText(
             @RequestParam(value = "source", required = false) List<String> sources,
             @RequestParam(value = "delimiters", required = false) String delimiters,
             @RequestParam(value = "minLength", required = false, defaultValue = "3") int minLength,
-            @RequestParam(value = "includeSubstrings", required = false, defaultValue = "true") boolean includeSubstrings
+            @RequestParam(value = "includeSubstrings", required = false, defaultValue = "true") boolean includeSubstrings,
+            @RequestParam(value = "includeObsoleteEntities", required = false, defaultValue = "false") boolean includeObsoleteEntities
     ) {
 
         if (!textTaggerService.isAvailable()) {
@@ -62,6 +63,10 @@ public HttpEntity<Map<String, Object>> tagText(
 
         List<TaggedEntity> entities = textTaggerService.tagText(text, ontologyIds, sources, delimiters, minLength, includeSubstrings);
 
+        if (!includeObsoleteEntities) {
+            entities.removeIf(e -> e.isObsolete);
+        }
+
         List<Map<String, Object>> entityMaps = new ArrayList<>(entities.size());
         for (TaggedEntity e : entities) {
             Map<String, Object> m = new LinkedHashMap<>();
@@ -73,6 +78,7 @@ public HttpEntity<Map<String, Object>> tagText(
             if (e.stringType != null) m.put("string_type", e.stringType);
             if (e.source != null) m.put("source", e.source);
             if (e.subjectCategories != null) m.put("subject_categories", e.subjectCategories);
+            if (e.isObsolete) m.put("is_obsolete", true);
             entityMaps.add(m);
         }
 

backend/src/main/java/uk/ac/ebi/spot/ols/service/TextTaggerService.java

@@ -44,10 +44,11 @@ public static class TaggedEntity {
         public final String stringType;
         public final String source;
         public final List<String> subjectCategories;
+        public final boolean isObsolete;
 
         public TaggedEntity(int start, int end, String termLabel, String termIri, String ontologyId,
                             String stringType, String source,
-                            List<String> subjectCategories) {
+                            List<String> subjectCategories, boolean isObsolete) {
             this.start = start;
             this.end = end;
             this.termLabel = termLabel;
@@ -56,6 +57,7 @@ public TaggedEntity(int start, int end, String termLabel, String termIri, String
             this.stringType = stringType;
             this.source = source;
             this.subjectCategories = subjectCategories;
+            this.isObsolete = isObsolete;
         }
     }
 
@@ -251,6 +253,7 @@ private List<TaggedEntity> parseResponse(String json) {
             String stringType = e.has("string_type") ? e.get("string_type").getAsString() : null;
             String source = e.has("source") ? e.get("source").getAsString() : null;
             List<String> subjectCategories = jsonArrayToStringList(e, "subject_categories");
+            boolean isObsolete = e.has("is_obsolete") && e.get("is_obsolete").getAsBoolean();
             results.add(new TaggedEntity(
                     e.get("start").getAsInt(),
                     e.get("end").getAsInt(),
@@ -259,7 +262,8 @@ private List<TaggedEntity> parseResponse(String json) {
                     e.has("ontology_id") ? e.get("ontology_id").getAsString() : "",
                     stringType,
                     source,
-                    subjectCategories
+                    subjectCategories,
+                    isObsolete
             ));
         }
         return results;

dataload/extract_strings_from_terms/src/main.rs

@@ -64,7 +64,7 @@ fn main() -> Result<(), Box<dyn Error>> {
     let mut total: u64 = 0;
     let mut embedded: u64 = 0;
 
-    writeln!(&mut writer, "pk\tontology_id\tentity_type\tiri\tlabel\thash\ttext_to_embed\tstring_type\tcurated_from_source\tcurated_from_subject_categories").unwrap();
+    writeln!(&mut writer, "pk\tontology_id\tentity_type\tiri\tlabel\thash\ttext_to_embed\tstring_type\tcurated_from_source\tcurated_from_subject_categories\tis_obsolete").unwrap();
 
     for input_file in &args.input_files {
         eprintln!("Processing file: {}", input_file);
@@ -167,6 +167,7 @@ fn process_entity(
     let mut labels:Vec<String> = Vec::new();
     let mut synonyms:Vec<String> = Vec::new();
     let mut is_defining_ontology = false;
+    let mut is_obsolete = false;
     let mut curated_from_entries: Vec<CuratedFromJson> = Vec::new();
 
     json.begin_object().unwrap();
@@ -184,6 +185,12 @@ fn process_entity(
             } else {
                 json.skip_value().unwrap();
             }
+        } else if key == "isObsolete" {
+            if json.peek().unwrap() == ValueType::Boolean {
+                is_obsolete = json.next_bool().unwrap();
+            } else {
+                json.skip_value().unwrap();
+            }
         } else if key == "curatedFrom" {
             curated_from_entries = parse_curated_from(json);
         } else {
@@ -210,12 +217,15 @@ fn process_entity(
     let pk = format!("{}:{}:{}", ontology_id, entity_type, &iri_value);
     let mut written: u64 = 0;
 
+    let is_obsolete_str = if is_obsolete { "true" } else { "" };
+
     // Emit LABEL rows (for labels and synonyms)
     for text in &texts_to_embed {
         if let Some(row) = make_row(text, tokenizer) {
-            writeln!(writer, "{}\t{}\t{}\t{}\t{}\t{}\t{}\tLABEL\t\t",
+            writeln!(writer, "{}\t{}\t{}\t{}\t{}\t{}\t{}\tLABEL\t\t\t{}",
                 pk, ontology_id, entity_type, &iri_value, &label_str,
-                row.hash, row.document
+                row.hash, row.document,
+                is_obsolete_str
             ).unwrap();
             written += 1;
         }
@@ -224,11 +234,12 @@ fn process_entity(
     // Emit CURATION rows (for curated text-to-term mappings)
     for entry in &curated_from_entries {
         if let Some(row) = make_row(&entry.text, tokenizer) {
-            writeln!(writer, "{}\t{}\t{}\t{}\t{}\t{}\t{}\tCURATION\t{}\t{}",
+            writeln!(writer, "{}\t{}\t{}\t{}\t{}\t{}\t{}\tCURATION\t{}\t{}\t{}",
                 pk, ontology_id, entity_type, &iri_value, &label_str,
                 row.hash, row.document,
                 entry.source,
-                entry.subject_categories
+                entry.subject_categories,
+                is_obsolete_str
             ).unwrap();
             written += 1;
         }

testcases/iri-labels/user-defined-pref-label.owl

@@ -66,6 +66,15 @@
         <rdfs:label xml:lang="en">XMPL000003</rdfs:label>
     </owl:Class>
 
+    <!-- http://exmpl.org/xmpl/XMPL000004 -->
+
+    <owl:Class rdf:about="http://exmpl.org/xmpl/XMPL000004">
+        <rdfs:subClassOf rdf:resource="http://exmpl.org/xmpl/XMPL000001"/>
+        <owl:deprecated rdf:datatype="http://www.w3.org/2001/XMLSchema#boolean">true</owl:deprecated>
+        <xmpl:Preferred_name xml:lang="en">obsolete xmpl entity</xmpl:Preferred_name>
+        <rdfs:label xml:lang="en">XMPL000004</rdfs:label>
+    </owl:Class>
+
 </rdf:RDF>
 
 

testcases_expected_output/iri-labels/use-user-defined-pref-label/autocomplete.jsonl

@@ -4,5 +4,6 @@
 {"ontologyId":"xmpl","id":"xmpl+class+http://exmpl.org/xmpl/XMPL000002","label":"label of the first children class"}
 {"ontologyId":"xmpl","id":"xmpl+class+http://exmpl.org/xmpl/XMPL000003","label":"2nd children class pref label"}
 {"ontologyId":"xmpl","id":"xmpl+class+http://exmpl.org/xmpl/XMPL000003","label":"label of the second children class"}
+{"ontologyId":"xmpl","id":"xmpl+class+http://exmpl.org/xmpl/XMPL000004","label":"obsolete xmpl entity"}
 {"ontologyId":"xmpl","id":"xmpl+property+http://exmpl.org/xmpl/Preferred_name","label":"XMPL_Preferred_name"}
 {"ontologyId":"xmpl","id":"xmpl+property+http://exmpl.org/xmpl/Synonym","label":"XMPL_Synonym"}