Skip to content

Commit 80f0a8d

Browse files
feragferag
authored
Merge pull request #257 from IFCA-Advanced-Computing/dev/evaluator_base
Creating EvaluatorBase
2 parents 885f3f0 + c59b878 commit 80f0a8d

File tree

12 files changed

+2587
-598
lines changed

12 files changed

+2587
-598
lines changed

‎api/evaluator.py

Lines changed: 133 additions & 239 deletions
Large diffs are not rendered by default.

‎api/rda.py

Lines changed: 16 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ def load_evaluator(wrapped_func):
2323
def wrapper(body, **kwargs):
2424
repo = body.get("repo")
2525
item_id = body.get("id", "")
26-
oai_base = body.get("oai_base")
26+
api_endpoint = body.get("api_endpoint")
2727
lang = body.get("lang", "en")
2828
pattern_to_query = body.get("q", "")
2929

@@ -36,24 +36,22 @@ def wrapper(body, **kwargs):
3636
# Get the identifiers through a search query
3737
ids = [item_id]
3838

39-
# FIXME oai-pmh should be no different
40-
downstream_logger = evaluator.logger
41-
if repo not in ["oai-pmh"]:
39+
downstream_logger = None
40+
try:
41+
logger.debug("Trying to import plugin from plugins.%s.plugin" % (repo))
42+
plugin = importlib.import_module("plugins.%s.plugin" % (repo), ".")
43+
downstream_logger = plugin.logger
44+
except Exception as e:
45+
logger.error(str(e))
46+
return str(e), 400
47+
if pattern_to_query:
4248
try:
43-
logger.debug("Trying to import plugin from plugins.%s.plugin" % (repo))
44-
plugin = importlib.import_module("plugins.%s.plugin" % (repo), ".")
45-
downstream_logger = plugin.logger
49+
ids = plugin.Plugin.get_ids(
50+
api_endpoint=api_endpoint, pattern_to_query=pattern_to_query
51+
)
4652
except Exception as e:
4753
logger.error(str(e))
4854
return str(e), 400
49-
if pattern_to_query:
50-
try:
51-
ids = plugin.Plugin.get_ids(
52-
oai_base=oai_base, pattern_to_query=pattern_to_query
53-
)
54-
except Exception as e:
55-
logger.error(str(e))
56-
return str(e), 400
5755

5856
# Set handler for evaluator logs
5957
evaluator_handler = ut.EvaluatorLogHandler()
@@ -66,11 +64,9 @@ def wrapper(body, **kwargs):
6664
result = {}
6765
exit_code = 200
6866
for item_id in ids:
69-
# FIXME oai-pmh should be no different
70-
if repo in ["oai-pmh"]:
71-
eva = evaluator.Evaluator(item_id, oai_base, lang, config=config_data)
72-
else:
73-
eva = plugin.Plugin(item_id, oai_base, lang, config=config_data)
67+
eva = plugin.Plugin(
68+
item_id, api_endpoint, lang, name=repo, config=config_data
69+
)
7470
_result, _exit_code = wrapped_func(body, eva=eva)
7571
logger.debug(
7672
"Raw result returned for indicator ID '%s': %s" % (item_id, _result)

‎api/utils.py

Lines changed: 128 additions & 133 deletions
Original file line numberDiff line numberDiff line change
@@ -128,26 +128,6 @@ def test_status(points):
128128
return test_status
129129

130130

131-
def oai_identify(oai_base):
132-
action = "?verb=Identify"
133-
return oai_request(oai_base, action)
134-
135-
136-
def oai_metadataFormats(oai_base):
137-
action = "?verb=ListMetadataFormats"
138-
xmlTree = oai_request(oai_base, action)
139-
metadataFormats = {}
140-
for e in xmlTree.findall(".//{http://www.openarchives.org/OAI/2.0/}metadataFormat"):
141-
metadataPrefix = e.find(
142-
"{http://www.openarchives.org/OAI/2.0/}metadataPrefix"
143-
).text
144-
namespace = e.find(
145-
"{http://www.openarchives.org/OAI/2.0/}metadataNamespace"
146-
).text
147-
metadataFormats[metadataPrefix] = namespace
148-
return metadataFormats
149-
150-
151131
def is_persistent_id(item_id):
152132
"""Returns boolean if the item id is or not a persistent identifier.
153133
@@ -448,119 +428,6 @@ def check_metadata_terms_with_values(metadata, terms):
448428
return df_access
449429

450430

451-
def oai_check_record_url(oai_base, metadata_prefix, pid):
452-
endpoint_root = urllib.parse.urlparse(oai_base).netloc
453-
try:
454-
pid_type = idutils.detect_identifier_schemes(pid)[0]
455-
except Exception as e:
456-
pid_type = "internal"
457-
logging.error(e)
458-
if pid_type != "internal":
459-
oai_pid = idutils.normalize_pid(pid, pid_type)
460-
else:
461-
oai_pid = pid
462-
action = "?verb=GetRecord"
463-
464-
test_id = "oai:%s:%s" % (endpoint_root, oai_pid)
465-
params = "&metadataPrefix=%s&identifier=%s" % (metadata_prefix, test_id)
466-
url_final = ""
467-
url = oai_base + action + params
468-
response = requests.get(url, verify=False, allow_redirects=True)
469-
logging.debug("Trying ID v1: url: %s | status: %i" % (url, response.status_code))
470-
error = 0
471-
for tags in ET.fromstring(response.text).findall(
472-
".//{http://www.openarchives.org/OAI/2.0/}error"
473-
):
474-
error = error + 1
475-
if error == 0:
476-
url_final = url
477-
478-
test_id = "%s" % (oai_pid)
479-
params = "&metadataPrefix=%s&identifier=%s" % (metadata_prefix, test_id)
480-
481-
url = oai_base + action + params
482-
logging.debug("Trying: " + url)
483-
response = requests.get(url, verify=False)
484-
error = 0
485-
for tags in ET.fromstring(response.text).findall(
486-
".//{http://www.openarchives.org/OAI/2.0/}error"
487-
):
488-
error = error + 1
489-
if error == 0:
490-
url_final = url
491-
492-
test_id = "%s:%s" % (pid_type, oai_pid)
493-
params = "&metadataPrefix=%s&identifier=%s" % (metadata_prefix, test_id)
494-
495-
url = oai_base + action + params
496-
logging.debug("Trying: " + url)
497-
response = requests.get(url, verify=False)
498-
error = 0
499-
for tags in ET.fromstring(response.text).findall(
500-
".//{http://www.openarchives.org/OAI/2.0/}error"
501-
):
502-
error = error + 1
503-
if error == 0:
504-
url_final = url
505-
506-
test_id = "oai:%s:%s" % (
507-
endpoint_root,
508-
oai_pid[oai_pid.rfind(".") + 1 : len(oai_pid)],
509-
)
510-
params = "&metadataPrefix=%s&identifier=%s" % (metadata_prefix, test_id)
511-
512-
url = oai_base + action + params
513-
logging.debug("Trying: " + url)
514-
response = requests.get(url, verify=False)
515-
error = 0
516-
for tags in ET.fromstring(response.text).findall(
517-
".//{http://www.openarchives.org/OAI/2.0/}error"
518-
):
519-
error = error + 1
520-
if error == 0:
521-
url_final = url
522-
523-
test_id = "oai:%s:b2rec/%s" % (
524-
endpoint_root,
525-
oai_pid[oai_pid.rfind(".") + 1 : len(oai_pid)],
526-
)
527-
params = "&metadataPrefix=%s&identifier=%s" % (metadata_prefix, test_id)
528-
529-
url = oai_base + action + params
530-
logging.debug("Trying: " + url)
531-
response = requests.get(url, verify=False)
532-
error = 0
533-
for tags in ET.fromstring(response.text).findall(
534-
".//{http://www.openarchives.org/OAI/2.0/}error"
535-
):
536-
error = error + 1
537-
if error == 0:
538-
url_final = url
539-
540-
return url_final
541-
542-
543-
def oai_get_metadata(url):
544-
logging.debug("Metadata from: %s" % url)
545-
oai = requests.get(url, verify=False, allow_redirects=True)
546-
try:
547-
xmlTree = ET.fromstring(oai.text)
548-
except Exception as e:
549-
logging.error("OAI_RQUEST: %s" % e)
550-
xmlTree = None
551-
return xmlTree
552-
553-
554-
def oai_request(oai_base, action):
555-
oai = requests.get(oai_base + action, verify=False) # Peticion al servidor
556-
try:
557-
xmlTree = ET.fromstring(oai.text)
558-
except Exception as e:
559-
logging.error("OAI_RQUEST: %s" % e)
560-
xmlTree = ET.fromstring("<OAI-PMH></OAI-PMH>")
561-
return xmlTree
562-
563-
564431
def find_dataset_file(metadata, url, data_formats):
565432
headers = {
566433
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36"
@@ -784,6 +651,31 @@ def check_standard_project_relation(value):
784651
return False
785652

786653

654+
def oai_request(oai_base, action):
655+
oai = requests.get(oai_base + action, verify=False) # Peticion al servidor
656+
try:
657+
xmlTree = ET.fromstring(oai.text)
658+
except Exception as e:
659+
logging.error("OAI_RQUEST: %s" % e)
660+
xmlTree = ET.fromstring("<OAI-PMH></OAI-PMH>")
661+
return xmlTree
662+
663+
664+
def oai_metadataFormats(oai_base):
665+
action = "?verb=ListMetadataFormats"
666+
xmlTree = oai_request(oai_base, action)
667+
metadataFormats = {}
668+
for e in xmlTree.findall(".//{http://www.openarchives.org/OAI/2.0/}metadataFormat"):
669+
metadataPrefix = e.find(
670+
"{http://www.openarchives.org/OAI/2.0/}metadataPrefix"
671+
).text
672+
namespace = e.find(
673+
"{http://www.openarchives.org/OAI/2.0/}metadataNamespace"
674+
).text
675+
metadataFormats[metadataPrefix] = namespace
676+
return metadataFormats
677+
678+
787679
def get_rdf_metadata_format(oai_base):
788680
rdf_schemas = []
789681
try:
@@ -797,6 +689,109 @@ def get_rdf_metadata_format(oai_base):
797689
return rdf_schemas
798690

799691

692+
def oai_check_record_url(oai_base, metadata_prefix, pid):
693+
endpoint_root = urllib.parse.urlparse(oai_base).netloc
694+
try:
695+
pid_type = idutils.detect_identifier_schemes(pid)[0]
696+
except Exception as e:
697+
pid_type = "internal"
698+
logging.error(e)
699+
if pid_type != "internal":
700+
oai_pid = idutils.normalize_pid(pid, pid_type)
701+
else:
702+
oai_pid = pid
703+
action = "?verb=GetRecord"
704+
705+
test_id = "oai:%s:%s" % (endpoint_root, oai_pid)
706+
params = "&metadataPrefix=%s&identifier=%s" % (metadata_prefix, test_id)
707+
url_final = ""
708+
url = oai_base + action + params
709+
response = requests.get(url, verify=False, allow_redirects=True)
710+
logging.debug("Trying ID v1: url: %s | status: %i" % (url, response.status_code))
711+
error = 0
712+
for tags in ET.fromstring(response.text).findall(
713+
".//{http://www.openarchives.org/OAI/2.0/}error"
714+
):
715+
error = error + 1
716+
if error == 0:
717+
url_final = url
718+
719+
test_id = "%s" % (oai_pid)
720+
params = "&metadataPrefix=%s&identifier=%s" % (metadata_prefix, test_id)
721+
722+
url = oai_base + action + params
723+
logging.debug("Trying: " + url)
724+
response = requests.get(url, verify=False)
725+
error = 0
726+
for tags in ET.fromstring(response.text).findall(
727+
".//{http://www.openarchives.org/OAI/2.0/}error"
728+
):
729+
error = error + 1
730+
if error == 0:
731+
url_final = url
732+
733+
test_id = "%s:%s" % (pid_type, oai_pid)
734+
params = "&metadataPrefix=%s&identifier=%s" % (metadata_prefix, test_id)
735+
736+
url = oai_base + action + params
737+
logging.debug("Trying: " + url)
738+
response = requests.get(url, verify=False)
739+
error = 0
740+
for tags in ET.fromstring(response.text).findall(
741+
".//{http://www.openarchives.org/OAI/2.0/}error"
742+
):
743+
error = error + 1
744+
if error == 0:
745+
url_final = url
746+
747+
test_id = "oai:%s:%s" % (
748+
endpoint_root,
749+
oai_pid[oai_pid.rfind(".") + 1 : len(oai_pid)],
750+
)
751+
params = "&metadataPrefix=%s&identifier=%s" % (metadata_prefix, test_id)
752+
753+
url = oai_base + action + params
754+
logging.debug("Trying: " + url)
755+
response = requests.get(url, verify=False)
756+
error = 0
757+
for tags in ET.fromstring(response.text).findall(
758+
".//{http://www.openarchives.org/OAI/2.0/}error"
759+
):
760+
error = error + 1
761+
if error == 0:
762+
url_final = url
763+
764+
test_id = "oai:%s:b2rec/%s" % (
765+
endpoint_root,
766+
oai_pid[oai_pid.rfind(".") + 1 : len(oai_pid)],
767+
)
768+
params = "&metadataPrefix=%s&identifier=%s" % (metadata_prefix, test_id)
769+
770+
url = oai_base + action + params
771+
logging.debug("Trying: " + url)
772+
response = requests.get(url, verify=False)
773+
error = 0
774+
for tags in ET.fromstring(response.text).findall(
775+
".//{http://www.openarchives.org/OAI/2.0/}error"
776+
):
777+
error = error + 1
778+
if error == 0:
779+
url_final = url
780+
781+
return url_final
782+
783+
784+
def oai_get_metadata(url):
785+
logger.debug("Metadata from: %s" % url)
786+
oai = requests.get(url, verify=False, allow_redirects=True)
787+
try:
788+
xmlTree = ET.fromstring(oai.text)
789+
except Exception as e:
790+
logger.error("OAI_RQUEST: %s" % e)
791+
xmlTree = None
792+
return xmlTree
793+
794+
800795
def licenses_list():
801796
url = "https://spdx.org/licenses/licenses.json"
802797
headers = {"Accept": "application/json"} # Type of response accpeted

0 commit comments

Comments
 (0)