Merge pull request #250 from IFCA-Advanced-Computing/feature/load_config

Simultaneously load main and plugin's config.ini

Author: ferag

api/evaluator.py

@@ -13,12 +13,8 @@
 import requests
 
 import api.utils as ut
-from fair import load_config
 
-logging.basicConfig(
-    stream=sys.stdout, level=logging.DEBUG, format="'%(name)s:%(lineno)s' | %(message)s"
-)
-logger = logging.getLogger(os.path.basename(__file__))
+logger = logging.getLogger("api.plugin.evaluation_steps")
 
 
 class ConfigTerms(property):
@@ -130,13 +126,13 @@ class Evaluator(object):
     lang : Language
     """
 
-    def __init__(self, item_id, oai_base=None, lang="en", plugin=None):
+    def __init__(self, item_id, oai_base=None, lang="en", plugin=None, config=None):
         self.item_id = item_id
         self.oai_base = oai_base
         self.metadata = None
         self.access_protocols = []
         self.cvs = []
-        self.config = load_config(plugin=plugin)
+        self.config = config
         # configuration terms
         self.terms_access_metadata = pd.DataFrame()
         self.terms_license_metadata = pd.DataFrame()

api/rda.py

@@ -15,9 +15,7 @@
 logging.basicConfig(
     stream=sys.stdout, level=logging.DEBUG, format="'%(name)s:%(lineno)s' | %(message)s"
 )
-logger = logging.getLogger(os.path.basename(__file__))
-
-config = load_config()
+logger = logging.getLogger("api")
 
 
 def load_evaluator(wrapped_func):
@@ -35,9 +33,9 @@ def wrapper(body, **kwargs):
             msg = "Neither the identifier nor the pattern to query was provided. Exiting.."
             logger.error(msg)
             return msg, 400
-
         # Get the identifiers through a search query
         ids = [item_id]
+
         # FIXME oai-pmh should be no different
         downstream_logger = evaluator.logger
         if repo not in ["oai-pmh"]:
@@ -61,15 +59,18 @@ def wrapper(body, **kwargs):
         evaluator_handler = ut.EvaluatorLogHandler()
         downstream_logger.addHandler(evaluator_handler)
 
+        # Load configuration
+        config_data = load_config(plugin=repo)
+
         # Collect FAIR checks per metadata identifier
         result = {}
         exit_code = 200
         for item_id in ids:
             # FIXME oai-pmh should be no different
             if repo in ["oai-pmh"]:
-                eva = evaluator.Evaluator(item_id, oai_base, lang)
+                eva = evaluator.Evaluator(item_id, oai_base, lang, config=config_data)
             else:
-                eva = plugin.Plugin(item_id, oai_base, lang)
+                eva = plugin.Plugin(item_id, oai_base, lang, config=config_data)
             _result, _exit_code = wrapped_func(body, eva=eva)
             logger.debug(
                 "Raw result returned for indicator ID '%s': %s" % (item_id, _result)
@@ -99,8 +100,8 @@ def endpoints(plugin=None, plugins_path="plugins"):
 
     # Obtain endpoint from each plugin's config
     for plug in plugins_list:
-        config = load_config(plugin=plug, fail_if_no_config=False)
-        endpoint = config.get("Generic", "endpoint", fallback="")
+        _config = load_config(plugin=plug, fail_if_no_config=False)
+        endpoint = _config.get("Generic", "endpoint", fallback="")
         if not endpoint:
             logger.debug(
                 "Plugin's config does not contain 'Generic:endpoint' section: %s" % plug
@@ -1339,7 +1340,7 @@ def rda_all(body, eva):
     result_points = 10
     num_of_tests = 10
 
-    generic_config = config["Generic"]
+    generic_config = eva.config["Generic"]
     api_config = os.path.join(
         app_dirname, generic_config.get("api_config", "fair-api.yaml")
     )

api/utils.py

@@ -12,7 +12,10 @@
 import requests
 from bs4 import BeautifulSoup
 
-logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+logging.basicConfig(
+    stream=sys.stdout, level=logging.DEBUG, format="'%(name)s:%(lineno)s' | %(message)s"
+)
+logger = logging.getLogger("api.utils")
 
 
 class EvaluatorLogHandler(logging.Handler):

fair.py

@@ -13,30 +13,30 @@
     level=logging.DEBUG,
     format="%(levelname)s:'%(name)s:%(lineno)s' | %(message)s",
 )
-
-logger = logging.getLogger(os.path.basename(__file__))
+logger = logging.getLogger("api")
 
 app_dirname = os.path.dirname(os.path.abspath(__file__))
 
 
-def load_config(plugin=None, fail_if_no_config=True):
+def load_config(plugin, fail_if_no_config=True):
+    config_file_main = os.path.join(app_dirname, "config.ini")
+    config_file_plugin = os.path.join(app_dirname, "plugins/%s/config.ini" % plugin)
     config = configparser.ConfigParser()
-    if plugin:
-        config_file = os.path.join(app_dirname, "plugins/%s/config.ini" % plugin)
-    else:
-        config_file = os.path.join(app_dirname, "config.ini")
-        if "CONFIG_FILE" in os.environ:
-            config_file = os.getenv("CONFIG_FILE")
-
     try:
-        config.read_file(open(config_file))
-        logging.debug("Main configuration successfully loaded: %s" % config_file)
+        config.read([config_file_main, config_file_plugin])
+        logging.debug(
+            "Successfully loaded main & plugin's configuration files (%s, %s)"
+            % (config_file_main, config_file_plugin)
+        )
     except FileNotFoundError as e:
         logging.error("Could not load config file: %s" % str(e))
         if fail_if_no_config:
             raise (e)
     except configparser.MissingSectionHeaderError as e:
-        message = "Could not find main config file: %s" % config_file
+        message = "Could not find any/all of the following config files: %s, %s" % (
+            config_file_main,
+            config_file_plugin,
+        )
         logging.error(message)
         logging.debug(e)
         error = {"code": 500, "message": "%s" % message}

plugins/digital_csic/config.ini

@@ -1,5 +1,5 @@
 [Generic]
-endpoint='http://digital.csic.es/dspace-oai/request'
+endpoint=http://digital.csic.es/dspace-oai/request
 [digital_csic]
 db_host =
 db_port =

plugins/digital_csic/plugin.py

@@ -21,8 +21,7 @@
 logging.basicConfig(
     stream=sys.stdout, level=logging.DEBUG, format="'%(name)s:%(lineno)s' | %(message)s"
 )
-
-logger = logging.getLogger(os.path.basename(__file__))
+logger = logging.getLogger("api.plugin")
 
 
 class ConfigTerms(property):
@@ -87,7 +86,7 @@ class Plugin(Evaluator):
     lang : Language
     """
 
-    def __init__(self, item_id, oai_base=None, lang="en"):
+    def __init__(self, item_id, oai_base=None, lang="en", config=None):
         logger.debug("Call parent")
         plugin = "digital_csic"
         super().__init__(item_id, oai_base, lang, plugin)

plugins/dspace7/plugin.py

@@ -16,7 +16,7 @@
 logging.basicConfig(
     stream=sys.stdout, level=logging.DEBUG, format="'%(name)s:%(lineno)s' | %(message)s"
 )
-logger = logging.getLogger(os.path.basename(__file__))
+logger = logging.getLogger("api.plugin")
 
 
 class DSpace_7(Evaluator):
@@ -32,7 +32,7 @@ class DSpace_7(Evaluator):
     lang : Language
     """
 
-    def __init__(self, item_id, oai_base=None, lang="en"):
+    def __init__(self, item_id, oai_base=None, lang="en", config=None):
         if oai_base == "":
             oai_base = None
         logger.debug("Call parent")

plugins/epos/plugin.py

@@ -18,12 +18,11 @@
 
 import api.utils as ut
 from api.evaluator import ConfigTerms, Evaluator
-from fair import load_config
 
 logging.basicConfig(
     stream=sys.stdout, level=logging.DEBUG, format="'%(name)s:%(lineno)s' | %(message)s"
 )
-logger = logging.getLogger(os.path.basename(__file__))
+logger = logging.getLogger("api.plugin")
 
 
 class Plugin(Evaluator):
@@ -46,15 +45,20 @@ class Plugin(Evaluator):
 
     name = "epos"
 
-    def __init__(self, item_id, oai_base=None, lang="en", config=None):
-        logger.debug("Creating instance of %s plugin" % self.name)
-        super().__init__(item_id, oai_base, lang, self.name)
-        # TO REDEFINE - WHICH IS YOUR PID TYPE?
-        self.id_type = "uuid"
-        global _
-        _ = super().translation()
+    def __init__(self, item_id, oai_base=None, lang="en", config=None, name="epos"):
+        # FIXME: Disable calls to parent class until a EvaluatorBase class is implemented
+        # super().__init__(item_id, oai_base, lang, self.name)
+        # global _
+        # _ = super().translation()
 
-        # You need a way to get your metadata in a similar format
+        self.name = name
+        self.item_id = item_id
+        self.api_endpoint = oai_base
+        self.config = config
+
+        logger.debug("Using FAIR-EVA's plugin: %s" % self.name)
+
+        # Metadata gathering
         metadata_sample = self.get_metadata()
         self.metadata = pd.DataFrame(
             metadata_sample,
@@ -148,7 +152,7 @@ def get_metadata(self):
         eml_schema = "epos"
 
         final_url = (
-            self.oai_base + "/resources/details/" + self.item_id + "?extended=true"
+            self.api_endpoint + "/resources/details/" + self.item_id + "?extended=true"
         )
 
         error_in_metadata = False

plugins/example_plugin/plugin.py

@@ -13,7 +13,7 @@
 logging.basicConfig(
     stream=sys.stdout, level=logging.DEBUG, format="'%(name)s:%(lineno)s' | %(message)s"
 )
-logger = logging.getLogger(os.path.basename(__file__))
+logger = logging.getLogger("api.plugin")
 
 
 class Plugin(Evaluator):
@@ -29,7 +29,7 @@ class Plugin(Evaluator):
     lang : Language
     """
 
-    def __init__(self, item_id, oai_base=None, lang="en"):
+    def __init__(self, item_id, oai_base=None, lang="en", config=None):
         plugin = "example_plugin"
         super().__init__(item_id, oai_base, lang, plugin)
         # TO REDEFINE - WHICH IS YOUR PID TYPE?

plugins/gbif/plugin.py

@@ -17,8 +17,7 @@
 logging.basicConfig(
     stream=sys.stdout, level=logging.DEBUG, format="'%(name)s:%(lineno)s' | %(message)s"
 )
-
-logger = logging.getLogger(os.path.basename(__file__))
+logger = logging.getLogger("api.plugin")
 
 
 class Plugin(Evaluator):
@@ -37,7 +36,7 @@ class Plugin(Evaluator):
     lang : Language
     """
 
-    def __init__(self, item_id, oai_base=None, lang="en"):
+    def __init__(self, item_id, oai_base=None, lang="en", config=None):
         logger.debug("Creating GBIF")
         plugin = "gbif"
         super().__init__(item_id, oai_base, lang, plugin)