Merge branch 'main' into dev/gbif_data

Author: ferag

.github/workflows/pre-commit.yml

@@ -11,4 +11,4 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - uses: actions/setup-python@v3
-    - uses: pre-commit/[email protected].0
+    - uses: pre-commit/[email protected].1

api/evaluator.py

@@ -292,48 +292,6 @@ def eval_uniqueness(self, id_list, data_or_metadata="(meta)data"):
 
         return (points, msg_list)
 
-    def eval_persistency(self, id_list, data_or_metadata="(meta)data"):
-        points = 0
-        msg_list = []
-        for _id in id_list:
-            _points = 0
-            if ut.is_persistent_id(_id):
-                _msg = "Found persistent identifier for the %s: %s" % (
-                    data_or_metadata,
-                    _id,
-                )
-                _points = 100
-                points = 100
-            else:
-                _msg = "Identifier is not persistent for the %s: %s" % (
-                    data_or_metadata,
-                    _id,
-                )
-            msg_list.append({"message": _msg, "points": _points})
-
-        return (points, msg_list)
-
-    def eval_uniqueness(self, id_list, data_or_metadata="(meta)data"):
-        points = 0
-        msg_list = []
-        for _id in id_list:
-            _points = 0
-            if ut.is_unique_id(_id):
-                _msg = "Found a globally unique identifier for the %s: %s" % (
-                    data_or_metadata,
-                    _id,
-                )
-                _points = 100
-                points = 100
-            else:
-                _msg = "Identifier found for the %s is not globally unique: %s" % (
-                    data_or_metadata,
-                    _id,
-                )
-            msg_list.append({"message": _msg, "points": _points})
-
-        return (points, msg_list)
-
     # TESTS
     #    FINDABLE
     @ConfigTerms(term_id="identifier_term")
@@ -1388,11 +1346,11 @@ def rda_i2_01m(self, **kwargs):
             term_data = kwargs["terms_cv"]
             term_metadata = term_data["metadata"]
 
-        for index, e_k in term_metadata.iterrows():
-            tmp_msg, cv = ut.check_controlled_vocabulary(e_k["text_value"])
-            if tmp_msg is not None:
-                logger.debug(_("Found potential vocabulary") + ": %s" % tmp_msg)
-                self.cvs.append(cv)
+            for index, e_k in term_metadata.iterrows():
+                tmp_msg, cv = ut.check_controlled_vocabulary(e_k["text_value"])
+                if tmp_msg is not None:
+                    logger.debug(_("Found potential vocabulary") + ": %s" % tmp_msg)
+                    self.cvs.append(cv)
 
         if len(self.cvs) > 0:
             for e in self.cvs:

api/rda.py

@@ -1,3 +1,4 @@
+import glob
 import importlib
 import logging
 import os
@@ -8,7 +9,7 @@
 from connexion import NoContent
 
 import api.utils as ut
-from api.evaluator import Evaluator
+from api import evaluator
 from fair import app_dirname, load_config
 
 logging.basicConfig(
@@ -38,10 +39,12 @@ def wrapper(body, **kwargs):
         # Get the identifiers through a search query
         ids = [item_id]
         # FIXME oai-pmh should be no different
+        downstream_logger = evaluator.logger
         if repo not in ["oai-pmh"]:
             try:
                 logger.debug("Trying to import plugin from plugins.%s.plugin" % (repo))
                 plugin = importlib.import_module("plugins.%s.plugin" % (repo), ".")
+                downstream_logger = plugin.logger
             except Exception as e:
                 logger.error(str(e))
                 return str(e), 400
@@ -54,25 +57,72 @@ def wrapper(body, **kwargs):
                     logger.error(str(e))
                     return str(e), 400
 
+        # Set handler for evaluator logs
+        evaluator_handler = ut.EvaluatorLogHandler()
+        downstream_logger.addHandler(evaluator_handler)
+
         # Collect FAIR checks per metadata identifier
         result = {}
         exit_code = 200
         for item_id in ids:
             # FIXME oai-pmh should be no different
             if repo in ["oai-pmh"]:
-                eva = Evaluator(item_id, oai_base, lang)
+                eva = evaluator.Evaluator(item_id, oai_base, lang)
             else:
                 eva = plugin.Plugin(item_id, oai_base, lang)
             _result, _exit_code = wrapped_func(body, eva=eva)
+            logger.debug(
+                "Raw result returned for indicator ID '%s': %s" % (item_id, _result)
+            )
             result[item_id] = _result
             if _exit_code != 200:
                 exit_code = _exit_code
 
+        # Append evaluator logs to the final results
+        result["evaluator_logs"] = evaluator_handler.logs
+        logger.debug("Evaluator logs appended through 'evaluator_logs' property")
+
         return result, exit_code
 
     return wrapper
 
 
+def endpoints(plugin=None, plugins_path="plugins"):
+    plugins_with_endpoint = []
+    links = []
+
+    # Get the list of plugins
+    modules = glob.glob(os.path.join(app_dirname, plugins_path, "*"))
+    plugins_list = [
+        os.path.basename(folder) for folder in modules if os.path.isdir(folder)
+    ]
+
+    # Obtain endpoint from each plugin's config
+    for plug in plugins_list:
+        config = load_config(plugin=plug, fail_if_no_config=False)
+        endpoint = config.get("Generic", "endpoint", fallback="")
+        if not endpoint:
+            logger.debug(
+                "Plugin's config does not contain 'Generic:endpoint' section: %s" % plug
+            )
+            logger.warning(
+                "Could not get (meta)data endpoint from plugin's config: %s " % plug
+            )
+        else:
+            logger.debug("Obtained endpoint for plugin '%s': %s" % (plug, endpoint))
+            links.append(endpoint)
+            plugins_with_endpoint.append(plug)
+    # Create a dict with all the found endpoints
+    enp = dict(zip(plugins_with_endpoint, links))
+    # If the plugin is given then only returns a message
+    if plugin:
+        try:
+            return enp[plugin]
+        except:
+            return (enp, 404)
+    return enp
+
+
 @load_evaluator
 def rda_f1_01m(body, eva):
     try:

api/utils.py

@@ -15,6 +15,15 @@
 logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
 
 
+class EvaluatorLogHandler(logging.Handler):
+    def __init__(self, level=logging.DEBUG):
+        self.level = level
+        self.logs = []
+
+    def handle(self, record):
+        self.logs.append("[%s] %s" % (record.levelname, record.msg))
+
+
 def get_doi_str(doi_str):
     doi_to_check = re.findall(
         r"10[\.-]+.[\d\.-]+/[\w\.-]+[\w\.-]+/[\w\.-]+[\w\.-]", doi_str
@@ -468,7 +477,7 @@ def oai_check_record_url(oai_base, metadata_prefix, pid):
 
     url = oai_base + action + params
     logging.debug("Trying: " + url)
-    response = requests.get(url)
+    response = requests.get(url, verify=False)
     error = 0
     for tags in ET.fromstring(response.text).findall(
         ".//{http://www.openarchives.org/OAI/2.0/}error"
@@ -482,7 +491,7 @@ def oai_check_record_url(oai_base, metadata_prefix, pid):
 
     url = oai_base + action + params
     logging.debug("Trying: " + url)
-    response = requests.get(url)
+    response = requests.get(url, verify=False)
     error = 0
     for tags in ET.fromstring(response.text).findall(
         ".//{http://www.openarchives.org/OAI/2.0/}error"
@@ -499,7 +508,7 @@ def oai_check_record_url(oai_base, metadata_prefix, pid):
 
     url = oai_base + action + params
     logging.debug("Trying: " + url)
-    response = requests.get(url)
+    response = requests.get(url, verify=False)
     error = 0
     for tags in ET.fromstring(response.text).findall(
         ".//{http://www.openarchives.org/OAI/2.0/}error"
@@ -516,7 +525,7 @@ def oai_check_record_url(oai_base, metadata_prefix, pid):
 
     url = oai_base + action + params
     logging.debug("Trying: " + url)
-    response = requests.get(url)
+    response = requests.get(url, verify=False)
     error = 0
     for tags in ET.fromstring(response.text).findall(
         ".//{http://www.openarchives.org/OAI/2.0/}error"
@@ -657,6 +666,11 @@ def check_controlled_vocabulary(value):
         if coar_c:
             cv_msg = "COAR - Controlled vocabulary. Data: %s" % coar_msg
             cv = "purl.org/coar"
+    elif "wikidata.org" in value:
+        wikidata_c, wikidata_msg = wikidata_check(value)
+        if wikidata_c:
+            cv_msg = "Wikidata - URI term. Data: %s" % wikidata_msg
+            cv = "wikidata.org/wiki"
     return cv_msg, cv
 
 
@@ -688,7 +702,7 @@ def orcid_basic_info(orcid):
     }
     try:
         url = "https://pub.orcid.org/v3.0/" + orcid
-        r = requests.get(url, headers=headers)  # GET with headers
+        r = requests.get(url, verify=False, headers=headers)  # GET with headers
         xmlTree = ET.fromstring(r.text)
         item = xmlTree.findall(
             ".//{http://www.orcid.org/ns/common}assertion-origin-name"
@@ -703,7 +717,7 @@ def orcid_basic_info(orcid):
 def loc_basic_info(loc):
     # Returns the first line of json LD
     headers = {"Accept": "application/json"}  # Type of response accpeted
-    r = requests.get(loc, headers=headers)  # GET with headers
+    r = requests.get(loc, verify=False, headers=headers)  # GET with headers
     output = r.json()
     return output[0]
 
@@ -715,7 +729,7 @@ def geonames_basic_info(geonames):
     geonames = geonames[0 : geonames.index("/")]
     url = "http://api.geonames.org/get?geonameId=%s&username=frames" % geonames
     headers = {"Accept": "application/json"}  # Type of response accpeted
-    r = requests.get(url, headers=headers)  # GET with headers
+    r = requests.get(url, verify=False, headers=headers)  # GET with headers
     logging.debug("Request genoames: %s" % r.text)
     output = ""
     try:
@@ -731,14 +745,24 @@ def coar_check(coar):
     coar = coar[0 : coar.index("/")]
     coar = coar.replace("resource_type", "resource_types")
     url = "https://vocabularies.coar-repositories.org/%s" % coar
-    r = requests.get(url)  # GET with headers
+    r = requests.get(url, verify=False)  # GET with headers
     logging.debug("Request coar: %s" % r.text)
     if r.status_code == 200:
         return True, "purl.org/coar"
     else:
         return False, ""
 
 
+def wikidata_check(wikidata):
+    logging.debug("Checking wikidata")
+    r = requests.head(wikidata, verify=False)  # GET with headers
+    logging.debug("Request coar: %s" % r.text)
+    if r.status_code == 200:
+        return True, "wikidata.org/wiki"
+    else:
+        return False, ""
+
+
 def getty_basic_info(loc):
     r = requests.get(loc + ".json")  # GET
     if r.status_code == 200:
@@ -773,7 +797,7 @@ def get_rdf_metadata_format(oai_base):
 def licenses_list():
     url = "https://spdx.org/licenses/licenses.json"
     headers = {"Accept": "application/json"}  # Type of response accpeted
-    r = requests.get(url, headers=headers)  # GET with headers
+    r = requests.get(url, verify=False, headers=headers)  # GET with headers
     output = r.json()
     licenses = []
     for e in output["licenses"]:
@@ -784,7 +808,7 @@ def licenses_list():
 def is_spdx_license(license_id, machine_readable=False):
     url = "https://spdx.org/licenses/licenses.json"
     headers = {"Accept": "application/json"}  # Type of response accpeted
-    r = requests.get(url, headers=headers)  # GET with headers
+    r = requests.get(url, verify=False, headers=headers)  # GET with headers
     payload = r.json()
     is_spdx = False
     license_list = []
@@ -819,19 +843,22 @@ def resolve_handle(handle_id):
 
     Returns:
     """
-    resolves = False
-    endpoint = urljoin("https://hdl.handle.net/api/", "handles/%s" % handle_id)
+    handle_id_normalized = idutils.normalize_doi(handle_id)
+    endpoint = urljoin(
+        "https://hdl.handle.net/api/", "handles/%s" % handle_id_normalized
+    )
     headers = {"Content-Type": "application/json"}
-    r = requests.get(endpoint, headers=headers)
+    r = requests.get(endpoint, verify=False, headers=headers)
     if not r.ok:
         msg = "Error while making a request to endpoint: %s (status code: %s)" % (
             endpoint,
             r.status_code,
         )
         raise Exception(msg)
-
     json_data = r.json()
     response_code = json_data.get("responseCode", -1)
+
+    resolves = False
     if response_code == 1:
         resolves = True
         msg = "Handle and associated values found (HTTP 200 OK)"
@@ -963,3 +990,15 @@ def check_fairsharing_abbreviation(fairlist, abreviation):
         if abreviation == standard["attributes"]["abbreviation"]:
             return (100, "Your metadata standard appears in Fairsharing")
     return (0, "Your metadata standard has not been found in Fairsharing")
+
+
+def check_ror(ror):
+    response = requests.get("https://api.ror.org/organizations/" + ror)
+
+    rordict = response.json()
+    name = rordict["name"]
+
+    if response.ok:
+        return (True, name)
+    else:
+        return (False, "")