marker set confidence added

Author: hkir-dev

src/ontology/bdso-edit.owl

@@ -45,6 +45,7 @@ Declaration(AnnotationProperty(<http://purl.obolibrary.org/obo/PCL_0010058>))
 Declaration(AnnotationProperty(<http://purl.obolibrary.org/obo/PCL_0010059>))
 Declaration(AnnotationProperty(<http://purl.obolibrary.org/obo/PCL_0010060>))
 Declaration(AnnotationProperty(<http://purl.obolibrary.org/obo/PCL_0010061>))
+Declaration(AnnotationProperty(<http://purl.obolibrary.org/obo/PCL_0010062>))
 Declaration(AnnotationProperty(dce:contributor))
 Declaration(AnnotationProperty(dce:description))
 Declaration(AnnotationProperty(dce:title))
@@ -102,6 +103,10 @@ AnnotationAssertion(rdfs:label <http://purl.obolibrary.org/obo/PCL_0010060> "cel
 
 AnnotationAssertion(rdfs:label <http://purl.obolibrary.org/obo/PCL_0010061> "nuclei_count")
 
+# Annotation Property: <http://purl.obolibrary.org/obo/PCL_0010062> (fbeta_confidence_score)
+
+AnnotationAssertion(rdfs:label <http://purl.obolibrary.org/obo/PCL_0010062> "fbeta_confidence_score")
+
 
 ############################
 #   Object Properties

src/patterns/data/default/CCN201912131_marker_set.tsv

@@ -22,6 +22,10 @@ data_vars:
   Marker_set_of: "xsd:string"
   Brain_region_abbv: "xsd:string"
   Species_abbv: "xsd:string"
+  FBeta_confidence_score: "xsd:double"
+
+annotationProperties:
+  fbetaConfidenceScore: "PCL:0010062"
 
 name:
    text: "NS forest marker set of %s %s (%s)."
@@ -46,3 +50,9 @@ logical_axioms:
 #     text: "'has_anatomical_context' some %s"
 #     vars:
 #        - Brain_region
+
+annotations:
+  - annotationProperty: fbetaConfidenceScore
+    text: "%s"
+    vars:
+      - FBeta_confidence_score

src/patterns/data/default/CCN201912132_marker_set.tsv

@@ -4,7 +4,7 @@
 import logging
 import os
 
-from template_generation_utils import get_root_nodes, read_taxonomy_config, generate_dendrogram_tree
+from template_generation_utils import get_root_nodes, read_taxonomy_config, generate_dendrogram_tree, index_dendrogram, read_csv_to_dict
 from dendrogram_tools import dend_json_2_nodes_n_edges
 from nomenclature_tools import nomenclature_2_nodes_n_edges
 
@@ -190,3 +190,56 @@ def generate_marker_table(marker_data, output_filepath):
         template.append(d)
     class_robot_template = pd.DataFrame.from_records(template)
     class_robot_template.to_csv(output_filepath.replace("CCN", "CS"), sep="\t", index=False)
+
+
+def get_nsforest_confidences(taxon, dend, ns_forest_marker_file):
+    """
+    Each NS forest marker files has a non-standard tabular structure. This function aligns cluster names mentioned in
+    the marker file with the dendrogram nodes and prepares a dict of accession_id - confidence_score.
+    Args:
+        taxon: taxonomy id
+        dend: dendrogram file
+        ns_forest_marker_file: path of the marker file
+    """
+    nomenclature_indexes = [
+                            index_dendrogram(dend, id_field_name="cell_set_preferred_alias", id_to_lower=True),
+                            index_dendrogram(dend, id_field_name="cell_set_accession", id_to_lower=True),
+                            index_dendrogram(dend, id_field_name="original_label", id_to_lower=True),
+                            index_dendrogram(dend, id_field_name="cell_set_additional_aliases", id_to_lower=True)
+                            ]
+
+    if taxon != "CS1908210":
+        nomenclature_indexes.append(index_dendrogram(dend, id_field_name="cell_set_aligned_alias", id_to_lower=True))
+
+    confidence_map = dict()
+
+    if os.path.isfile(ns_forest_marker_file):
+        headers, raw_marker_data = read_csv_to_dict(ns_forest_marker_file, id_column_name="clusterName")
+    else:
+        # human mtg file extension is outlier
+        headers, raw_marker_data = read_csv_to_dict(str(ns_forest_marker_file).replace(".csv", ".tsv"),
+                                                    id_column_name="clusterName", delimiter="\t")
+
+    for cluster_name in raw_marker_data:
+        cluster_name_variants = [cluster_name.lower(), cluster_name.lower().replace("-", "/"),
+                                 cluster_name.replace("Micro", "Microglia").lower(),
+                                 ("(Mouse " + cluster_name + ")-like").lower(),
+                                 ("(Mouse " + cluster_name.replace("-", "/") + ")-like").lower()]
+
+        nomenclature_node = search_terms_in_index(cluster_name_variants, nomenclature_indexes)
+        if nomenclature_node:
+            node_id = nomenclature_node["cell_set_accession"]
+            confidence_map[node_id] = raw_marker_data[cluster_name]["f-measure"]
+        else:
+            raise ValueError("Node with cluster name '{}' couldn't be found in the nomenclature of {}."
+                             .format(cluster_name, taxon))
+
+    return confidence_map
+
+
+def search_terms_in_index(term_variants, indexes):
+    for term in term_variants:
+        for index in indexes:
+            if term in index:
+                return index[term]
+    return None

src/patterns/data/default/CCN202002013_marker_set.tsv

@@ -10,6 +10,7 @@
     extract_taxonomy_name_from_path
 from nomenclature_tools import nomenclature_2_nodes_n_edges
 from pcl_id_factory import get_class_id, get_individual_id, get_taxonomy_id, get_dataset_id, get_marker_gene_set_id
+from marker_tools import get_nsforest_confidences
 
 log = logging.getLogger(__name__)
 
@@ -28,6 +29,7 @@
 ALLEN_DESCRIPTIONS_PATH = "{}/{}/All Descriptions_{}.json"
 DATASET_INFO_CSV = "{}/{}/{}_landingpage_dataset_info.csv"
 TAXONOMY_INFO_CSV = "{}/{}/{}_Taxonomy_Info_Panel.csv"
+NSFOREST_MARKER_CSV = "{}/NSForestMarkers/{}_{}_NSForest_Markers.csv"
 
 EXPRESSION_SEPARATOR = "|"
 
@@ -513,7 +515,7 @@ def generate_datasets_template(centralized_data_folder, output_filepath):
                          .format(taxonomy_id, expected_file_name))
 
 
-def generate_marker_gene_set_template(taxonomy_file_path, output_filepath):
+def generate_marker_gene_set_template(taxonomy_file_path, centralized_data_folder, output_filepath):
     taxon = extract_taxonomy_name_from_path(taxonomy_file_path)
     taxonomy_config = read_taxonomy_config(taxon)
 
@@ -529,17 +531,22 @@ def generate_marker_gene_set_template(taxonomy_file_path, output_filepath):
             gene_db_path = ENSEMBLE_PATH.format(str(taxonomy_config["Reference_gene_list"][0]).strip().lower())
             gene_names = read_gene_data(gene_db_path)
             minimal_markers = read_markers(MARKER_PATH.format(taxon.replace("CCN", "").replace("CS", "")), gene_names)
-
         else:
             minimal_markers = {}
 
+        ns_forest_marker_file = NSFOREST_MARKER_CSV.format(centralized_data_folder,
+                                                           taxonomy_config['Species_abbv'][0],
+                                                           taxonomy_config['Brain_region_abbv'][0])
+        confidences = get_nsforest_confidences(taxon, dend, ns_forest_marker_file)
+
         class_seed = ['defined_class',
                       'Marker_set_of',
                       'Minimal_markers',
                       'Brain_region_abbv',
                       'Species_abbv',
                       'Brain_region',
-                      'Parent'
+                      'Parent',
+                      'FBeta_confidence_score'
                       ]
         class_template = []
 
@@ -557,6 +564,8 @@ def generate_marker_gene_set_template(taxonomy_file_path, output_filepath):
                         d['Species_abbv'] = taxonomy_config['Species_abbv'][0]
                     d['Brain_region'] = taxonomy_config['Brain_region'][0]
                     d['Parent'] = "SO:0001260"  # sequence collection
+                    if o['cell_set_accession'] in confidences:
+                        d['FBeta_confidence_score'] = confidences[o['cell_set_accession']]
 
                     for k in class_seed:
                         if not (k in d.keys()):

src/patterns/data/default/CS1908210_marker_set.tsv

@@ -58,6 +58,6 @@
     elif args.tx:
         generate_taxonomies_template(args.input, args.output)
     elif args.ms:
-        generate_marker_gene_set_template(args.input, args.output)
+        generate_marker_gene_set_template(args.input, args.input2, args.output)
     else:
         generate_ind_template(args.input, args.input2, args.output)

src/patterns/dosdp-patterns/taxonomy_marker_set.yaml

@@ -0,0 +1,55 @@
+import unittest
+import os
+from marker_tools import get_nsforest_confidences
+from dendrogram_tools import dend_json_2_nodes_n_edges
+from nomenclature_tools import nomenclature_2_nodes_n_edges
+
+current_dir = os.path.dirname(os.path.realpath(__file__))
+PATH_MOUSE_NOMENCLATURE = os.path.join(current_dir, "../dendrograms/nomenclature_table_CCN202002013.csv")
+PATH_HUMAN_M1_NOMENCLATURE = os.path.join(current_dir, "../dendrograms/nomenclature_table_CCN201912131.csv")
+PATH_MARMOSET_M1_NOMENCLATURE = os.path.join(current_dir, "../dendrograms/nomenclature_table_CCN201912132.csv")
+PATH_HUMAN_MTG_NOMENCLATURE = os.path.join(current_dir, "../dendrograms/CS1908210.json")
+PATH_CENTRALIZED_DATA = os.path.join(current_dir, "./test_data/centralized_data/MOp_taxonomies_ontology/")
+
+class MarkerToolsTest(unittest.TestCase):
+
+    def test_get_nsforest_confidences_mouse(self):
+        dend = nomenclature_2_nodes_n_edges(PATH_MOUSE_NOMENCLATURE)
+        marker_path = os.path.join(PATH_CENTRALIZED_DATA, "NSForestMarkers/Mouse_MOp_NSForest_Markers.csv")
+        confidences = get_nsforest_confidences("CCN202002013", dend, marker_path)
+
+        self.assertEqual("0.988920549", confidences["CS202002013_179"])
+        self.assertEqual("0.607476636", confidences["CS202002013_3"])
+
+    def test_get_nsforest_confidences_human_m1(self):
+        dend = nomenclature_2_nodes_n_edges(PATH_HUMAN_M1_NOMENCLATURE)
+        marker_path = os.path.join(PATH_CENTRALIZED_DATA, "NSForestMarkers/Human_M1_NSForest_Markers.csv")
+        confidences = get_nsforest_confidences("CS201912131", dend, marker_path)
+
+        self.assertEqual("0.989133387", confidences["CS201912131_150"])  # Glutamatergic
+        self.assertEqual("0.689189189", confidences["CS201912131_69"])  # Inh L5-6 PVALB GAPDHP60
+        self.assertEqual("0.825020189", confidences["CS201912131_134"])  # (Mouse L5 IT)-like
+        self.assertFalse("CS201912131_145" in confidences)  # (Mouse IT projecting)-like
+        self.assertFalse("CS201912131_162" in confidences)  # (Mouse Vip)-like_C4
+
+    def test_get_nsforest_confidences_marmoset_m1(self):
+        dend = nomenclature_2_nodes_n_edges(PATH_MARMOSET_M1_NOMENCLATURE)
+        marker_path = os.path.join(PATH_CENTRALIZED_DATA, "NSForestMarkers/Marmoset_M1_NSForest_Markers.csv")
+        confidences = get_nsforest_confidences("CS201912132", dend, marker_path)
+
+        self.assertEqual("0.428954424", confidences["CS201912132_79"])  # Astro FGFR3 EPHB1
+        self.assertEqual("0.759294697", confidences["CS201912132_104"])  # (Mouse L6 CT)-like
+        self.assertEqual("0.7227388", confidences["CS201912132_108"])  # Peri
+        self.assertEqual("0.790527018", confidences["CS201912132_109"])  # VLMC
+        self.assertEqual("0.889261745", confidences["CS201912132_93"])  # VLMC SLC1A3-like SLC47A1
+        self.assertFalse("CS201912132_113" in confidences)  # (Mouse Non-IT projecting)-like
+        self.assertFalse("CS201912132_135" in confidences)  # All Cells
+
+    def test_get_nsforest_confidences_human_mtg(self):
+        dend = dend_json_2_nodes_n_edges(PATH_HUMAN_MTG_NOMENCLATURE)
+        marker_path = os.path.join(PATH_CENTRALIZED_DATA, "NSForestMarkers/Human_MTG_NSForest_Markers.tsv")
+        confidences = get_nsforest_confidences("CS1908210", dend, marker_path)
+
+        self.assertEqual("0.55", confidences["CS1908210011"])  # Inh L1-2 VIP TSPAN12
+        self.assertEqual("0.58", confidences["CS1908210059"])  # Exc L4-6 RORB C1R
+        self.assertEqual("0.91", confidences["CS1908210075"])  # Micro L1-6 TYROBP