extractor: new per-project base extractor

This change creates a new base extractor that is aware of the project
that we are getting the accounting for. This way the code gets much
simpler, as it is not needed to keep track of projects, clients, etc.
within the extrating methods. As a side effect, this change breaks the
old ceilometer extractor.

Author: alvarolopez

caso/extract/base.py

@@ -82,6 +82,9 @@
 @six.add_metaclass(abc.ABCMeta)
 class BaseExtractor(object):
     def __init__(self):
+        # FIXME(aloga): add deprecated warning here
+        LOG.warning("Using DEPRECATED extractor!! Please update as soon as "
+                    "possible!!")
         try:
             mapping = json.loads(open(CONF.mapping_file).read())
         except ValueError:
@@ -134,3 +137,51 @@ def extract_for_project(self, project, extract_from):
 
         This method should be overriden in a subclass.
         """
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseProjectExtractor(object):
+    def __init__(self, project):
+        self.project = project
+
+        # FIXME(remove this)
+        try:
+            mapping = json.loads(open(CONF.mapping_file).read())
+        except ValueError:
+            # FIXME(aloga): raise a proper exception here
+            raise
+        else:
+            self.voms_map = {}
+            for vo, vomap in six.iteritems(mapping):
+                tenant = vomap.get("tenant", None)
+                tenants = vomap.get("tenants", [])
+                if tenant is not None:
+                    warnings.warn("Using deprecated 'tenant' mapping, please "
+                                  "use 'projects' instead", DeprecationWarning)
+                if tenants:
+                    warnings.warn("Using deprecated 'tenants' mapping, please "
+                                  "use 'projects' instead", DeprecationWarning)
+                tenants.append(tenant)
+                projects = vomap.get("projects", tenants)
+                if not projects:
+                    LOG.warning("No project mapping found for VO %s" % vo)
+                for project in projects:
+                    self.voms_map[project] = vo
+
+    def vm_status(self, status):
+        """Return the status corresponding to the OpenStack status.
+
+        :param status: OpenStack status.
+        """
+        return openstack_vm_statuses.get(status.lower(), 'unknown')
+
+    @abc.abstractmethod
+    def extract(self, extract_from):
+        """Extract records for a project from given date.
+
+        :param extract_from: datetime.datetime object indicating the date to
+                             extract records from
+        :returns: A dictionary of {"server_id": caso.record.Record"}
+
+        This method should be overriden in a subclass.
+        """

caso/extract/manager.py

@@ -22,6 +22,7 @@
 from oslo_log import log
 import six
 
+from caso.extract import base
 from caso import loading
 
 cli_opts = [
@@ -65,7 +66,7 @@
 class Manager(object):
     def __init__(self):
         extractor = loading.get_available_extractors()[CONF.extractor]
-        self.extractor = extractor()
+        self.extractor = extractor
         self.last_run_base = os.path.join(CONF.spooldir, "lastrun")
 
     def get_lastrun(self, project):
@@ -114,7 +115,6 @@ def get_records(self):
 
         cloud_records = {}
         ip_records_total = {}
-        extr = self.extractor
         for project in CONF.projects:
             LOG.info("Extracting records for project '%s'" % project)
 
@@ -127,9 +127,19 @@ def get_records(self):
             LOG.debug("Extracting records from '%s'" % extract_from)
             LOG.debug("Extracting records to '%s'" % extract_to)
             try:
-                records, ip_records = extr.extract_for_project(project,
-                                                               extract_from,
-                                                               extract_to)
+                if issubclass(self.extractor, base.BaseProjectExtractor):
+                    extractor = self.extractor(project)
+                    records, ip_records = extractor.extract(
+                        extract_from,
+                        extract_to
+                    )
+                else:
+                    extractor = self.extractor
+                    records, ip_records = extractor.extract_for_project(
+                        project,
+                        extract_from,
+                        extract_to
+                    )
             except Exception:
                 LOG.exception("Cannot extract records for '%s', got "
                               "the following exception: " % project)