Load VO mapping in correct place

alvarolopez · alvarolopez · commit 66b7bd7b9998 · 2023-04-10T15:56:00.000+02:00
diff --git a/caso/_cmd/projects.py b/caso/_cmd/projects.py
@@ -30,8 +30,8 @@ def main():
     caso.config.parse_args(sys.argv)
     log.setup(cfg.CONF, "caso")
     manager = caso.manager.Manager()
-    for prj in manager.projects():
-        print(prj)
+    for prj, vo in manager.projects_and_vos():
+        print(f"'{prj} mapped to VO '{vo}'")
 
 
 if __name__ == "__main__":
diff --git a/caso/extract/base.py b/caso/extract/base.py
@@ -17,8 +17,6 @@
 """Module containing the base class and configuration for all cASO extractors."""
 
 import abc
-import json
-import warnings
 
 from oslo_config import cfg
 from oslo_log import log
@@ -29,16 +27,6 @@
     cfg.StrOpt(
         "service_name", default="$site_name", help="Service name within the site"
     ),
-    cfg.StrOpt(
-        "mapping_file",
-        default="/etc/caso/voms.json",
-        deprecated_group="extractor",
-        deprecated_for_removal=True,
-        deprecated_reason="This option is marked for removal in the next release. "
-        "Please see the release notes, and migrate your current configuration "
-        "to use the project_mapping file as soon as possible.",
-        help="File containing the VO <-> project mapping as used in Keystone-VOMS.",
-    ),
 ]
 
 CONF = cfg.CONF
@@ -55,40 +43,6 @@ def __init__(self, project):
         """Initialize extractor, loading the VO map."""
         self.project = project
 
-    @property
-    def voms_map(self):
-        """Get the VO map."""
-        # FIXME(remove this)
-        try:
-            mapping = json.loads(open(CONF.mapping_file).read())
-        except ValueError:
-            # FIXME(aloga): raise a proper exception here
-            raise
-        else:
-            voms_map = {}
-            for vo, vomap in six.iteritems(mapping):
-                tenant = vomap.get("tenant", None)
-                tenants = vomap.get("tenants", [])
-                if tenant is not None:
-                    warnings.warn(
-                        "Using deprecated 'tenant' mapping, please "
-                        "use 'projects' instead",
-                        DeprecationWarning,
-                    )
-                if tenants:
-                    warnings.warn(
-                        "Using deprecated 'tenants' mapping, please "
-                        "use 'projects' instead",
-                        DeprecationWarning,
-                    )
-                tenants.append(tenant)
-                projects = vomap.get("projects", tenants)
-                if not projects:
-                    LOG.warning(f"No project mapping found for VO {vo}")
-                for project in projects:
-                    voms_map[project] = vo
-            return voms_map
-
     @abc.abstractmethod
     def extract(self, extract_from):
         """Extract records for a project from given date.
diff --git a/caso/extract/manager.py b/caso/extract/manager.py
@@ -17,8 +17,10 @@
 """Module containing the manager for all extractors configured in cASO."""
 
 import datetime
+import json
 import os.path
 import sys
+import warnings
 
 import dateutil.parser
 from dateutil import tz
@@ -43,6 +45,21 @@
         default="caso",
         help="Tag used to mark a project in Keystone to be extracted by cASO",
     ),
+    cfg.StrOpt(
+        "vo_property",
+        default="accounting:VO",
+        help="Property key used to get the VO name from the project properties. ",
+    ),
+    cfg.StrOpt(
+        "mapping_file",
+        default="/etc/caso/voms.json",
+        deprecated_group="extractor",
+        deprecated_for_removal=True,
+        deprecated_reason="This option is marked for removal in the next release. "
+        "Please see the release notes, and migrate your current configuration "
+        "to use the project_mapping file as soon as possible.",
+        help="File containing the VO <-> project mapping as used in Keystone-VOMS.",
+    ),
     cfg.StrOpt(
         "extract-to",
         deprecated_name="extract_to",
@@ -97,6 +114,7 @@ def __init__(self):
         self.extractors = extractors
         self.last_run_base = os.path.join(CONF.spooldir, "lastrun")
 
+        self._voms_map = {}
         self.keystone = self._get_keystone_client()
 
     @property
@@ -145,6 +163,70 @@ def write_lastrun(self, project):
         with open(lfile, "w") as fd:
             fd.write(str(datetime.datetime.now(tz.tzutc())))
 
+    @property
+    def voms_map(self):
+        """Get the VO map."""
+        if self._voms_map:
+            return self._voms_map
+
+        try:
+            mapping = json.loads(open(CONF.mapping_file).read())
+        except ValueError:
+            # FIXME(aloga): raise a proper exception here
+            raise
+        else:
+            self._voms_map = {}
+            for vo, vomap in six.iteritems(mapping):
+                tenant = vomap.get("tenant", None)
+                tenants = vomap.get("tenants", [])
+                if tenant is not None:
+                    warnings.warn(
+                        "Using deprecated 'tenant' mapping, please "
+                        "use 'projects' instead",
+                        DeprecationWarning,
+                    )
+                if tenants:
+                    warnings.warn(
+                        "Using deprecated 'tenants' mapping, please "
+                        "use 'projects' instead",
+                        DeprecationWarning,
+                    )
+                tenants.append(tenant)
+                projects = vomap.get("projects", tenants)
+                if not projects:
+                    LOG.warning(f"No project mapping found for VO {vo}")
+                for project in projects:
+                    self._voms_map[project] = vo
+            return self._voms_map
+
+    def get_project_vo(self, project_id):
+        """Get the VO where the project should be mapped."""
+        project = self.keystone.projects.get(project_id)
+        project.get()
+        vo = project.to_dict().get(CONF.vo_property, None)
+        if vo is None:
+            LOG.warning(
+                f"No mapping could be found for project '{project_id}' in the "
+                "Keystone project metadata, please check cASO documentation."
+            )
+            vo = self.voms_map.get(project_id, None)
+            if vo is None:
+                LOG.warning(
+                    "No mapping could be found for project "
+                    f"'{project_id}', please check mapping file!"
+                )
+            else:
+                LOG.warning(
+                    "Using deprecated mapping file, please check cASO documentation "
+                    "and migrate to Keystone properties as soon as possible."
+                )
+        else:
+            LOG.debug(
+                f"Found VO mapping ({vo}) in Keystone project '{project_id}' "
+                "metadata."
+            )
+        return vo
+
     def get_records(self):
         """Get records from given date.
 
@@ -173,6 +255,8 @@ def get_records(self):
         for project in self.projects:
             LOG.info(f"Extracting records for project '{project}'")
 
+            vo = self.get_project_vo(project)
+
             extract_from = CONF.extract_from or self.get_lastrun(project)
             if isinstance(extract_from, six.string_types):
                 extract_from = dateutil.parser.parse(extract_from)
@@ -196,7 +280,7 @@ def get_records(self):
                     f"({extract_from} to {extract_to})"
                 )
                 try:
-                    extractor = extractor_cls(project)
+                    extractor = extractor_cls(project, vo)
                     records = extractor.extract(extract_from, extract_to)
                     current_count = len(records)
                     record_count += current_count
diff --git a/caso/extract/openstack/base.py b/caso/extract/openstack/base.py
@@ -36,11 +36,6 @@
         "there are several defined in the OpenStack site. "
         "Defaults to None.",
     ),
-    cfg.StrOpt(
-        "vo_property",
-        default="accounting:VO",
-        help="Property key used to get the VO name from the project properties. ",
-    ),
 ]
 
 CONF.register_opts(opts)
@@ -51,14 +46,14 @@
 class BaseOpenStackExtractor(base.BaseProjectExtractor):
     """Base OpenStack Extractor that all other extractors should inherit from."""
 
-    def __init__(self, project):
+    def __init__(self, project, vo):
         """Initialize the OpenStack extractor for a given project."""
         super(BaseOpenStackExtractor, self).__init__(project)
 
         self.keystone = self._get_keystone_client()
         self.project_id = self._get_project_id()
 
-        self.vo = self._get_vo()
+        self.vo = vo
 
         class Users:
             def __init__(self, parent):
@@ -114,34 +109,6 @@ def _get_keystone_user(self, uuid):
             LOG.exception(e)
             return None
 
-    def _get_vo(self):
-        """Get the VO where the project should be mapped."""
-        project = self.keystone.projects.get(self.project)
-        project.get()
-        vo = project.to_dict().get(CONF.vo_property, None)
-        if vo is None:
-            LOG.warning(
-                f"No mapping could be found for project '{self.project}' in the "
-                "Keystone project metadata, please check cASO documentation."
-            )
-            vo = self.voms_map.get(self.project, None)
-            if vo is None:
-                LOG.warning(
-                    "No mapping could be found for project "
-                    f"'{self.project}', please check mapping file!"
-                )
-            else:
-                LOG.warning(
-                    "Using deprecated mapping file, please check cASO documentation "
-                    "and migrate to Keystone properties as soon as possible."
-                )
-        else:
-            LOG.debug(
-                f"Found VO mapping ({vo}) in Keystone project '{self.project}' "
-                "metadata."
-            )
-        return vo
-
     # FIXME(aloga): this has to go inside a record
     @staticmethod
     def _get_measure_time():
diff --git a/caso/extract/openstack/cinder.py b/caso/extract/openstack/cinder.py
@@ -37,9 +37,9 @@
 class CinderExtractor(base.BaseOpenStackExtractor):
     """An OpenStack Volume (Cinder) record extractor for cASO."""
 
-    def __init__(self, project):
+    def __init__(self, project, vo):
         """Get a Cinder record extractor for a given project."""
-        super(CinderExtractor, self).__init__(project)
+        super(CinderExtractor, self).__init__(project, vo)
 
         self.cinder = self._get_cinder_client()
 
diff --git a/caso/extract/openstack/neutron.py b/caso/extract/openstack/neutron.py
@@ -38,9 +38,9 @@
 class NeutronExtractor(base.BaseOpenStackExtractor):
     """An OpenStack Network (Neutron) record extractor for cASO."""
 
-    def __init__(self, project):
+    def __init__(self, project, vo):
         """Get a Neutron record extractor for a given project."""
-        super(NeutronExtractor, self).__init__(project)
+        super(NeutronExtractor, self).__init__(project, vo)
 
         self.neutron = self._get_neutron_client()
 
diff --git a/caso/extract/openstack/nova.py b/caso/extract/openstack/nova.py
@@ -100,9 +100,9 @@
 class NovaExtractor(base.BaseOpenStackExtractor):
     """An OpenStack Compute (Nova) record extractor for cASO."""
 
-    def __init__(self, project):
+    def __init__(self, project, vo):
         """Get a Nova record extractor for a given project."""
-        super(NovaExtractor, self).__init__(project)
+        super(NovaExtractor, self).__init__(project, vo)
 
         self.nova = self._get_nova_client()
         self.glance = self._get_glance_client()
diff --git a/caso/manager.py b/caso/manager.py
@@ -21,6 +21,7 @@
 
 from oslo_concurrency import lockutils
 from oslo_config import cfg
+from oslo_log import log
 
 import caso.extract.manager
 from caso import loading
@@ -65,6 +66,8 @@
 CONF.register_opts(opts)
 CONF.register_cli_opts(cli_opts)
 
+LOG = log.getLogger(__name__)
+
 
 class Manager(object):
     """cASO manager class to deal with the main functionality."""
@@ -90,6 +93,16 @@ def projects(self):
 
         return self.extractor_manager.projects
 
+    def projects_and_vos(self):
+        """Get the configured projects and VOs as tuples."""
+        self._load_managers()
+
+        for prj in self.extractor_manager.projects:
+            try:
+                yield (prj, self.extractor_manager.get_project_vo(prj))
+            except Exception as e:
+                LOG.error(e)
+
     def run(self):
         """Run the manager.
 
diff --git a/caso/tests/extract/test_manager.py b/caso/tests/extract/test_manager.py
@@ -75,6 +75,7 @@ def test_extract(self):
         ret = self.manager.get_records()
         self.m_extractor.assert_called_once_with(
             "bazonk",
+            mock.ANY,
         )
         self.m_extractor.return_value.extract.assert_called_once_with(
             dateutil.parser.parse(extract_from).replace(tzinfo=tz.tzutc()),
@@ -109,6 +110,7 @@ def test_get_records_with_lastrun(self):
             m.assert_called_once_with("bazonk")
             self.m_extractor.assert_called_once_with(
                 "bazonk",
+                mock.ANY,
             )
             self.m_extractor.return_value.extract.assert_called_once_with(
                 dateutil.parser.parse(lastrun).replace(tzinfo=tz.tzutc()),
diff --git a/etc/caso/caso.conf.sample b/etc/caso/caso.conf.sample
@@ -37,6 +37,10 @@
 # Tag used to mark a project in Keystone to be extracted by cASO (string value)
 #caso_tag = caso
 
+# Property key used to get the VO name from the project properties.  (string
+# value)
+#vo_property = accounting:VO
+
 # DEPRECATED: File containing the VO <-> project mapping as used in Keystone-
 # VOMS. (string value)
 # This option is deprecated for removal.
