Improving GBIF metrics and dealing with exceptions

Author: ferag

api/evaluator.py

@@ -1428,6 +1428,7 @@ def rda_i3_01m(self, **kwargs):
             if row["text_value"].split("/")[-1] not in self.item_id:
                 id_list.append(row["text_value"])
         points, msg_list = self.eval_persistency(id_list)
+        return (points, msg_list)
 
     def rda_i3_01d(self):
         """Indicator RDA-A1-01M.
@@ -1854,14 +1855,16 @@ def rda_r1_3_01d(self, **kwargs):
         terms_reusability_richness_list = terms_reusability_richness["list"]
         terms_reusability_richness_metadata = terms_reusability_richness["metadata"]
 
-        element = terms_reusability_richness_metadata.loc[
-            terms_reusability_richness_metadata["element"].isin(["availableFormats"]),
-            "text_value",
-        ].values[0]
-        for form in element:
-            availableFormats.append(form["label"])
-
         try:
+            element = terms_reusability_richness_metadata.loc[
+                terms_reusability_richness_metadata["element"].isin(
+                    ["availableFormats"]
+                ),
+                "text_value",
+            ].values[0]
+            for form in element:
+                availableFormats.append(form["label"])
+
             f = open(path)
             f.close()
 

api/utils.py

@@ -710,10 +710,10 @@ def orcid_basic_info(orcid):
         item = xmlTree.findall(
             ".//{http://www.orcid.org/ns/common}assertion-origin-name"
         )
+        basic_info = "ORCID Name: %s" % item[0].text
     except Exception as e:
         logging.error(e)
         return basic_info
-    basic_info = "ORCID Name: %s" % item[0].text
     return basic_info
 
 

plugins/gbif/config.ini

@@ -37,33 +37,29 @@ terms_findability_richness = ['Title',
                               'Format',
                               'Identifier',
                               'Language']
+
 [gbif]
 # (meta)data terms to find the resource identifier
-identifier_term = [['alternateIdentifier','']]
-identifier_term_data = [['alternateIdentifier','']]
+identifier_term = [['dataset','alternateIdentifier']]
+identifier_term_data = [['dataset','alternateIdentifier']]
 
 # Metadata terms to check richness (generic). These terms should be included [term, qualifier]. None means no qualifier
-terms_quali_generic = [['contributor',None],
-                       ['date', None],
-                       ['description', None],
-                       ['identifier', None],
-                       ['publisher', None],
-                       ['rights', None],
-                       ['title', None],
-                       ['subject', None]]
+terms_quali_generic = [['dataset.creator', 'givenName'],
+                       ['dataset.creator', 'surName'],
+                       ['dataset', 'pubDate'],
+                       ['dataset.abstract', 'para'],
+                       ['dataset.intellectualRights.para.ulink', 'citetitle'],
+                       ['dataset', 'title'],
+                       ['dataset.keywordSet', 'keyword']]
 
 # Metadata terms to check richness (disciplinar). These terms should be included [term, qualifier]
-terms_quali_disciplinar = [['contributor', None],
-                           ['date', None],
-                           ['description', None],
-                           ['identifier', None],
-                           ['publisher', None],
-                           ['rights', None],
-                           ['title', None],
-                           ['subject', None]]
+terms_quali_disciplinar = [['dataset.coverage.geographicCoverage', 'geographicDescription'],
+                           ['dataset.coverage.temporalCoverage.rangeOfDates.beginDate', 'calendarDate'],
+                           ['dataset.coverage.temporalCoverage.rangeOfDates.endDate', 'calendarDate'],
+                           ['dataset.coverage.taxonomicCoverage.taxonomicClassification', 'taxonRankValue']]
 
 # Metadata terms that defines accessibility (case sensitive)
-terms_access = [['access', ''], ['rights', '']]
+terms_access = [['dataset.intellectualRights.para.ulink', 'citetitle']]
 
 # Metadata terms to check discoverability richness.
 #
@@ -84,37 +80,37 @@ terms_access = [['access', ''], ['rights', '']]
 # Format                File format                             availableFormats
 # Identifier            Data Unique ID                          DOI
 # Language              NA                                      NA
-terms_findability_richness = [['title',''],
-                              ['keywords',''],
-                              ['description',''],
-                              ['type','relatedDataProducts'],
-                              ['paths', 'spatial'],
-                              ['temporalCoverage','relatedDataProducts'],
-                              ['dataProvider','relatedDataProducts'],
-                              ['license',''],
-                              ['availableFormats',''],
-                              ['identifiers','relatedDataProducts']]
+terms_findability_richness = [['dataset', 'title']],
+                              ['dataset.keywordSet', 'keyword'],
+                              ['dataset.abstract', 'para'],
+                              ['dataset.coverage.geographicCoverage', 'geographicDescription'],
+                              ['dataset.coverage.temporalCoverage.rangeOfDates.beginDate', 'calendarDate'],
+                              ['dataset.coverage.temporalCoverage.rangeOfDates.endDate', 'calendarDate'],
+                              ['dataset.intellectualRights.para.ulink', 'citetitle'],
+                              ['dataset','alternateIdentifier']]
 
 # Metadata terms to check reusability richness
-terms_reusability_richness = [['rigths',''],
-                              ['license','']]
-
+terms_reusability_richness = [['dataset','alternateIdentifier'],
+                              ['additionalMetadata.metadata.gbif', 'hierarchyLevel']]
 
 
 # Metadata terms wich includes controlled vocabularies. More controlled vocabularies can be imlpemented in plugins
-terms_cv = [['coverage', 'spatial'], ['subject', 'lcsh']]
+terms_cv = [['dataset.creator', 'userId']]
 
 # List of data formats that are standard for the community
 supported_data_formats = [".txt", ".pdf", ".csv", ".nc", ".doc", ".xls", ".zip", ".rar", ".tar", ".png", ".jpg"]
 
 # Metadata terms that defines links or relation with authors, contributors (preferebly in ORCID format)
-terms_qualified_references = [['contributor', None]]
+terms_qualified_references = [['dataset.creator', 'userId'],
+                              ['dataset.contact', 'userId'],
+                              ['dataset.project.personnel', 'userId'],
+                              ['dataset.metadataProvider', 'userId' ]]
 
 # Metadata terms that defines links or relation with other resources, (preferebly in ORCID format, URIs or persistent identifiers)
-terms_relations = [['relation', None]]
+terms_relations = [['dataset.creator', 'userId']]
 
 # Metadata terms that defines the license type
-terms_license = [['rights', '']]
+terms_license = [['dataset.intellectualRights.para.ulink', 'citetitle']]
 
 # Metadata terms that defines metadata about provenance
 terms_provenance =[['curationAndProvenanceObligations','']]
@@ -123,7 +119,7 @@ terms_provenance =[['curationAndProvenanceObligations','']]
 terms_access_protocols =['http','https','ftp']
 
 # Manual metadata access
-metadata_access_manual = ['https://github.com/epos-eu/Hands-On-EPOS-API']
+metadata_access_manual = ['https://techdocs.gbif.org/en/openapi/']
 
 # Manual data access
 data_access_manual = ['https://techdocs.gbif.org/en/openapi/']
@@ -153,9 +149,9 @@ terms_vocabularies=[['identifiers','relatedDataProducts'],
                    ['license',''],
                    ['contactPoints','relatedDataProducts']]
 
-api_mail = [email protected]
-api_user = ma8
-api_pass = safdsaRY
+api_mail =
+api_user =
+api_pass =
 
 
 [fairsharing]

plugins/gbif/plugin.py

@@ -12,7 +12,7 @@
 import requests
 
 from api.evaluator import Evaluator
-from plugins.gbif.gbif_data import ICA, gbif_doi_download
+from plugins.gbif.gbif_data import ICA, gbif_doi_download, gbif_doi_search
 
 logging.basicConfig(
     stream=sys.stdout, level=logging.DEBUG, format="'%(name)s:%(lineno)s' | %(message)s"
@@ -114,149 +114,65 @@ def get_metadata(self):
             final_url = final_url.replace("www.gbif.org/", "api.gbif.org/v1/")
             final_url = final_url + "/document"
         response = requests.get(final_url, verify=False)
-        tree = ET.fromstring(response.text)
-
-        print("gbif5")
-        eml_schema = "{eml://ecoinformatics.org/eml-2.1.1}"
-        metadata_sample = []
-        elementos = tree.find(".//")
-        for e in elementos:
-            if e.text != "" or e.text != "\n    " or e.text != "\n":
-                metadata_sample.append([eml_schema, e.tag, e.text, None])
-            for i in e.iter():
-                if len(list(i.iter())) > 0:
-                    for se in i.iter():
-                        metadata_sample.append(
-                            [eml_schema, e.tag + "." + i.tag, se.text, se.tag]
-                        )
-                elif i.tag != e.tag and (
-                    i.text != "" or i.text != "\n    " or i.text != "\n"
-                ):
-                    metadata_sample.append([eml_schema, e.tag, i.text, i.tag])
+
+        def print_hierarchy_with_qualifier(elem, namespace, metadata_sample, path=""):
+            parts = path.split(".")
+            md_schema = parts[0]
+            quali = parts[-1]
+            if len(elem) == 0 and elem.text != None:
+                # Si el elemento no tiene hijos, y tiene un padre, lo guardamos en qualifier
+                if path:
+                    qualifier = f"{path}.{elem.tag}"
+                    metadata_sample.append(
+                        [
+                            namespace,
+                            path.replace(namespace + ".", ""),
+                            elem.text,
+                            elem.tag,
+                        ]
+                    )
+            else:
+                # Si tiene hijos, seguimos recorriendo la jerarquía
+                new_path = f"{path}.{elem.tag}" if path else elem.tag
+                for child in elem:
+                    print_hierarchy_with_qualifier(
+                        child, namespace, metadata_sample, new_path
+                    )
+
+        def parse_and_print_xml(response):
+            tree = ET.fromstring(response.text)
+            namespace = tree.tag
+            metadata_sample = []
+            print_hierarchy_with_qualifier(tree, namespace, metadata_sample)
+            return pd.DataFrame(
+                metadata_sample,
+                columns=["metadata_schema", "element", "text_value", "qualifier"],
+            )
+
+        metadata_sample = parse_and_print_xml(response)
+
         return metadata_sample
 
     def rda_a1_01m(self):
         # IF your ID is not an standard one (like internal), this method should be redefined
         points = 0
         msg = "Data is not accessible"
+        data_res = gbif_doi_search(self.item_id)
+        if len(data_res) > 0:
+            points = 100
+            msg = "Data found"
+
         return (points, msg)
 
     def rda_a1_02m(self):
         # IF your ID is not an standard one (like internal), this method should be redefined
         points = 0
         msg = "Data is not accessible"
-        return (points, msg)
-
-    def rda_i1_02m(self):
-        """Indicator RDA-A1-01M
-        This indicator is linked to the following principle: I1: (Meta)data use a formal, accessible,
-        shared, and broadly applicable language for knowledge representation. More information
-        about that principle can be found here.
-
-        This indicator focuses on the machine-understandability aspect of the metadata. This means
-        that metadata should be readable and thus interoperable for machines without any
-        requirements such as specific translators or mappings.
-
-        Technical proposal:
-
-        Parameters
-        ----------
-        item_id : str
-            Digital Object identifier, which can be a generic one (DOI, PID), or an internal (e.g. an
-            identifier from the repo)
-
-        Returns
-        -------
-        points
-            A number between 0 and 100 to indicate how well this indicator is supported
-        msg
-            Message with the results or recommendations to improve this indicator
-        """
-
-        # TO REDEFINE
-        points = 0
-        msg = "No machine-actionable metadata format found. OAI-PMH endpoint may help"
-        return (points, msg)
-
-    def rda_i1_02d(self):
-        """Indicator RDA-A1-01M
-        This indicator is linked to the following principle: I1: (Meta)data use a formal, accessible,
-        shared, and broadly applicable language for knowledge representation. More information
-        about that principle can be found here.
-
-        This indicator focuses on the machine-understandability aspect of the data. This means that
-        data should be readable and thus interoperable for machines without any requirements such
-        as specific translators or mappings.
-
-        Technical proposal:
-
-        Parameters
-        ----------
-        item_id : str
-            Digital Object identifier, which can be a generic one (DOI, PID), or an internal (e.g. an
-            identifier from the repo)
-
-        Returns
-        -------
-        points
-            A number between 0 and 100 to indicate how well this indicator is supported
-        msg
-            Message with the results or recommendations to improve this indicator
-        """
-        return self.rda_i1_02m()
-
-    def rda_r1_3_01m(self):
-        """Indicator RDA-A1-01M
-        This indicator is linked to the following principle: R1.3: (Meta)data meet domain-relevant
-        community standards.
-
-        This indicator requires that metadata complies with community standards.
-
-        Technical proposal:
-
-        Parameters
-        ----------
-        item_id : str
-            Digital Object identifier, which can be a generic one (DOI, PID), or an internal (e.g. an
-            identifier from the repo)
-
-        Returns
-        -------
-        points
-            A number between 0 and 100 to indicate how well this indicator is supported
-        msg
-            Message with the results or recommendations to improve this indicator
-        """
-        # TO REDEFINE
-        points = 0
-        msg = _(
-            "Currently, this repo does not include community-bsed schemas. If you need to include yours, please contact."
-        )
-        return (points, msg)
-
-    def rda_r1_3_01d(self):
-        """Indicator RDA_R1.3_01D.
-
-        Technical proposal:
+        data_res = gbif_doi_search(self.item_id)
+        if len(data_res) > 0:
+            points = 100
+            msg = "Data found"
 
-        Parameters
-        ----------
-        item_id : str
-            Digital Object identifier, which can be a generic one (DOI, PID), or an internal (e.g. an
-            identifier from the repo)
-
-        Returns
-        -------
-        points
-            A number between 0 and 100 to indicate how well this indicator is supported
-        msg
-            Message with the results or recommendations to improve this indicator
-        """
-        # TO REDEFINE
-        points = 0
-        msg = _(
-            "Currently, this repo does not include community-bsed schemas. If you need to include yours, please contact."
-        )
         return (points, msg)
 
     def data_01(self):