Merge pull request #214 from ImperialCollegeLondon/213-improve-rendering-of-metadata-on-zenodo-for-sequenced-taxonomy-sheets

Improve rendering of Zenodo metadata for sequenced taxonomy sheets

Author: jacobcook1995

safedata_validator/taxa.py

@@ -48,7 +48,9 @@
     blank_value,
 )
 
-GBIF_BACKBONE_RANKS = [
+ALL_BACKBONE_RANKS = [
+    "domain",
+    "superkingdom",
     "kingdom",
     "phylum",
     "class",
@@ -59,19 +61,16 @@
     "subspecies",
 ]
 
-# Possible top level ranks one of which must be provided
-SEQ_TOP_RANKS = ["domain", "superkingdom", "kingdom"]
+# GBIF backbone doesn't include the two highest-level ranks ("domain", "superkingdom")
+GBIF_BACKBONE_RANKS = ALL_BACKBONE_RANKS[2:]
 
-# List of additional ranks (in descending order) can be used to describe the taxonomy
-# further
-SEQ_ADDITIONAL_RANKS = [
-    "phylum",
-    "class",
-    "order",
-    "family",
-    "genus",
-    "species",
-]
+# For sequenced taxonomy the top level rank can be any of the top three ranks ("domain",
+# "superkingdom", "kingdom")
+SEQ_TOP_RANKS = ALL_BACKBONE_RANKS[0:3]
+
+# Sequenced taxonomy can then also include anything else as a backbone rank apart from
+# subspecies
+SEQ_ADDITIONAL_RANKS = ALL_BACKBONE_RANKS[3:-1]
 
 # NBCI name regex
 NCBI_prefix_re = re.compile("^[a-z]__")
@@ -1392,7 +1391,10 @@ def repeat_names(self) -> set[str]:
 
 
 def taxon_index_to_text(
-    taxa: list[dict], html: bool = False, indent_width: int = 4
+    taxa: list[dict],
+    html: bool = False,
+    indent_width: int = 4,
+    lowest_taxa: str | None = None,
 ) -> str | tags.div:
     """Render a taxon index as text or html.
 
@@ -1404,6 +1406,8 @@ def taxon_index_to_text(
         taxa: A list of taxon dictionaries containing the taxa for a dataset.
         html: Render as html or text.
         indent_width: The indentation width to use for successive taxonomic ranks.
+        lowest_taxa: The lowest taxonomic rank that the index renders, if no rank is
+            provided then the index is rendered for all ranks.
 
     Returns:
         Either a HTML or text representation of the taxa tree.
@@ -1468,8 +1472,26 @@ def _format_name(tx: dict, use_html: bool = html):
             if first_nm != taxon["worksheet_name"]:
                 surp_tx_ids.append(idx)
 
+    # Eliminate any taxa with ranks below the minimum
+    if lowest_taxa:
+        # Check that the lowest rank appears in the full set of taxa
+        if lowest_taxa not in ALL_BACKBONE_RANKS:
+            raise ValueError(
+                f"Rank provided to render taxa tree down to {lowest_taxa} is not a "
+                f"backbone rank! Should be one of: {ALL_BACKBONE_RANKS}"
+            )
+
+        # Generate the full list of ranks that should be rendered
+        rendered_ranks = ALL_BACKBONE_RANKS[: ALL_BACKBONE_RANKS.index(lowest_taxa) + 1]
+
+        # Then add any taxa that have ranks that aren't in the list of rendered ranks to
+        # the superfluous taxa index
+        for idx, taxon in enumerate(taxa):
+            if taxon["taxon_rank"] not in rendered_ranks:
+                surp_tx_ids.append(idx)
+
     # Delete taxa that are superfluous by index
-    for index in sorted(surp_tx_ids, reverse=True):
+    for index in sorted(set(surp_tx_ids), reverse=True):
         del taxa[index]
 
     # group taxa by their parent id

safedata_validator/templates/description_template.html

@@ -141,11 +141,25 @@ <h2>GBIF taxa details</h2>
   <h2>Bioinformatics taxonomy details</h2>
 
   <p>The dataset contains taxa that were identified as part of a bioinformatics
-    workkflow. The taxa listed in the dataset are compiled to provide the taxonomic
-    summary shown below. These taxa are not independently validated as they have been
-    generated using established reference databases.
+    workflow. These taxa are not independently validated as they have been generated using
+    established reference databases. Details of the reference databases used by each
+    sequenced taxonomy worksheet are provided below. Taxonomy trees (resolved to phylum
+    level) are also provided. For more detailed information on the taxa contained in the
+    dataset, you will need to download the data.
   </p>
-  {{ seq_taxa }}
+  {% for sheet_name, details in seq_taxa.items() %}
+  <h3>{{ sheet_name }} taxonomy sheet</h3>
+  <h4>Reference taxonomy database details</h4>
+  <ul>
+    <li>Reference taxonomy database: {{ details.database_name }}</li>
+    <li>Database version used: {{ details.database_version }}</li>
+    {% if details.database_link is not none %}
+    <li>Database link: <a href="{{details.database_link}}">{{ details.database_link }}</a></li>
+    {% endif %}
+  </ul>
+  <h4>Taxa tree (to phylum level)</h4>
+  <p>{{details.index}}</p>
+  {% endfor %}
   {% endif %}
 
   {% endif %}

safedata_validator/zenodo.py

@@ -684,19 +684,17 @@ def dataset_description(
     sequenced_taxa_sheets = dataset_metadata["sequenced_taxa"]
     if sequenced_taxa_sheets:
         context_dict["seq_taxa"] = {}
-        context_dict["seq_taxa_metadata"] = {}
         for sheet_name, taxon_data in sequenced_taxa_sheets.items():
-            context_dict["seq_taxa"][f"{sheet_name}"] = taxon_index_to_text(
-                taxa=taxon_data["taxon_index"], html=True
-            )
             context_dict["seq_taxa"][f"{sheet_name}"] = {
                 "database_name": taxon_data["database_name"],
                 "database_version": taxon_data["database_version"],
                 "database_link": taxon_data["database_link"],
+                "index": taxon_index_to_text(
+                    taxa=taxon_data["taxon_index"], html=True, lowest_taxa="phylum"
+                ),
             }
     else:
         context_dict["seq_taxa"] = None
-        context_dict["seq_taxa_metadata"] = None
 
     html = template.render(context_dict)
 

test/conftest.py

@@ -9,6 +9,7 @@
 import certifi
 import openpyxl
 import pytest
+import simplejson
 from dotmap import DotMap
 
 from safedata_validator.field import Dataset, DataWorksheet
@@ -47,7 +48,7 @@ def fixture_files():
         ("good_seq_taxa_file", "Test_format_good_Seq.xlsx"),
         ("bad_seq_taxa_file", "Test_format_bad_Seq.xlsx"),
         ("good_seq_file_dataset_json", "Test_format_good_Seq.json"),
-        ("good_seq_file_zenodo_json", "zenodo_214116.json"),
+        ("good_seq_file_zenodo_json", "zenodo_292625.json"),
     ]
 
     real_files = {ky: os.path.join(fixture_dir, vl) for ky, vl in real_files}
@@ -305,6 +306,16 @@ def example_seq_files(config_filesystem, request):
         return wb
 
 
+@pytest.fixture()
+def example_dataset_metadata(config_filesystem):
+    """Fixture providing an example dataset metadata to use in testing."""
+
+    with open(FIXTURE_FILES.rf.good_seq_file_dataset_json) as f:
+        dataset_metadata = simplejson.load(f)
+
+    return dataset_metadata
+
+
 @pytest.fixture()
 def fixture_gbif_validator(fixture_resources):
     """Fixture to return GBIF taxon validators."""