Let operators mark projects directly on keystone

This change allows to mark a project directly on Keystone with a
configurable tag in order to be extracted. The list of projects will be
merged with the configured projects in caso.conf

Sem-Ver: feature

Author: alvarolopez

caso/_cmd/projects.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2014 Spanish National Research Council (CSIC)
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sys
+
+from oslo_config import cfg
+from oslo_log import log
+
+import caso.config
+import caso.manager
+
+CONF = cfg.CONF
+
+
+def main():
+    """Get cASO configured projects."""
+    caso.config.parse_args(sys.argv)
+    log.setup(cfg.CONF, "caso")
+    manager = caso.manager.Manager()
+    for prj in manager.projects():
+        print(prj)
+
+
+if __name__ == "__main__":
+    main()

caso/extract/base.py

@@ -33,7 +33,14 @@
         "projects",
         default=[],
         deprecated_name="tenants",
-        help="List of projects to extract accounting records from.",
+        help="List of projects to extract accounting records from. You can "
+        "use this option, or add 'caso' tag to the project in Keystone. "
+        "Please refer to the documentation for more details.",
+    ),
+    cfg.StrOpt(
+        "caso_tag",
+        default="caso",
+        help="Tag used to mark a project in Keystone to be extracted by cASO",
     ),
     cfg.StrOpt(
         "mapping_file",

caso/extract/manager.py

@@ -26,6 +26,7 @@
 from oslo_log import log
 import six
 
+from caso import keystone_client
 from caso import loading
 
 cli_opts = [
@@ -65,6 +66,7 @@
 
 CONF.register_cli_opts(cli_opts)
 CONF.import_opt("projects", "caso.extract.base")
+CONF.import_opt("caso_tag", "caso.extract.base")
 
 LOG = log.getLogger(__name__)
 
@@ -84,6 +86,20 @@ def __init__(self):
         self.extractors = extractors
         self.last_run_base = os.path.join(CONF.spooldir, "lastrun")
 
+        self.keystone = self._get_keystone_client()
+
+    @property
+    def projects(self):
+        """Get list of configured projects."""
+        projects = CONF.projects
+        aux = [i.id for i in self.keystone.projects.list(tags=CONF.caso_tag)]
+        return set(projects + aux)
+
+    def _get_keystone_client(self):
+        """Get a Keystone Client to get the projects that we will use."""
+        client = keystone_client.get_client(CONF, system_scope="all")
+        return client
+
     def get_lastrun(self, project):
         """Get lastrun file for a given project."""
         lfile = f"{self.last_run_base}.{project}"
@@ -143,7 +159,7 @@ def get_records(self):
             extract_to = now
 
         all_records = []
-        for project in CONF.projects:
+        for project in self.projects:
             LOG.info(f"Extracting records for project '{project}'")
 
             extract_from = CONF.extract_from or self.get_lastrun(project)

caso/keystone_client.py

@@ -34,11 +34,11 @@
 opts += loading.get_auth_plugin_conf_options("password")
 
 
-def get_session(conf, project):
+def get_session(conf, project, system_scope=None):
     """Get an auth session."""
     # First try using project_id
     auth_plugin = loading.load_auth_from_conf_options(
-        conf, CFG_GROUP, project_id=project
+        conf, CFG_GROUP, project_id=project, system_scope=system_scope
     )
     sess = loading.load_session_from_conf_options(conf, CFG_GROUP, auth=auth_plugin)
     try:
@@ -52,7 +52,7 @@ def get_session(conf, project):
     return sess
 
 
-def get_client(conf, project):
+def get_client(conf, project=None, system_scope=None):
     """Return a client for Keystone."""
-    sess = get_session(conf, project)
+    sess = get_session(conf, project, system_scope)
     return ks_client_v3.Client(session=sess, interface="public")

caso/manager.py

@@ -78,6 +78,18 @@ def __init__(self):
 
         self.lock_path = CONF.lock_path
 
+    def _load_managers(self):
+        # Load the managers here to have the config options loaded and
+        # available
+        self.extractor_manager = caso.extract.manager.Manager()
+        self.messenger = caso.messenger.Manager()
+
+    def projects(self):
+        """Get the configured projects."""
+        self._load_managers()
+
+        return self.extractor_manager.projects
+
     def run(self):
         """Run the manager.
 
@@ -86,10 +98,7 @@ def run(self):
             - Gets all records from the configured extractors
             - Pushes all the records to the messengers
         """
-        # Load the managers here to have the config options loaded and
-        # available
-        self.extractor_manager = caso.extract.manager.Manager()
-        self.messenger = caso.messenger.Manager()
+        self._load_managers()
 
         @lockutils.synchronized(
             "caso_should_not_run_in_parallel", lock_path=self.lock_path, external=True

caso/tests/extract/test_manager.py

@@ -39,11 +39,18 @@ def setUp(self):
         self.m_extractor = mock.MagicMock()
         self.m_extractor.return_value.extract.return_value = self.records
         patched.return_value = {"mock": self.m_extractor}
+
+        self.p_keystone = mock.patch(
+            "caso.extract.manager.Manager._get_keystone_client"
+        )
+        self.p_keystone.start()
+
         self.manager = manager.Manager()
 
     def tearDown(self):
         """Run after each test, reset state and environment."""
         self.p_extractors.stop()
+        self.p_keystone.stop()
         self.reset_flags()
 
         super(TestCasoManager, self).tearDown()

doc/source/configuration.rst

@@ -66,6 +66,26 @@ that you are applying the correct changes as listed in the following table.
 |             | Modified | ``“identity:get_user”: “rule:admin_or_owner or role:accounting”`` |
 +-------------+----------+-------------------------------------------------------------------+
 
+Selecting projects to get usages
+================================
+
+``cASO`` will extract project usages for those projects that have been explictly marked
+by the operator by either of the ways explained below. The final project list will
+result from the merge of both methods, so thay are not mutually exclusive.
+
+* Tagging the project with the configured ``caso_tag`` in OpenStack Keystone. By default
+  this option is set to ``caso``, so in order to mark a project to get extracted you
+  should use the following command for each of the projects::
+
+    openstack project set --tag caso <project id>
+
+  You can check the list of projects to get usages by using::
+
+    openstack project list --tags caso
+
+* Using the ``projects`` list in the ``[DEFAULT]`` section of your configuration file
+  (see below).
+
 cASO configuration
 ==================
 
@@ -91,7 +111,11 @@ of every option. You should check at least the following options:
   records from. You can use either the project ID or the project name. We
   recommend that you use the project ID, especially if you are using
   domain-based authentication, as otherwise gathering the information might
-  fail.
+  fail. This option, and the usage of ``caso_tag`` below will set up the final
+  project list.
+* ``caso_tag`` (default value: ``caso``), specified the tag to be used filter
+  projects to extract their usage. The projects that are listed with this tag,
+  as well as the ``projects`` list set above will set up the final project list.
 * ``messengers`` (list, default: ``noop``). List of the messengers to publish
   data to. Records will be pushed to all these messengers, in order. Valid
   messengers shipped with cASO are:

etc/caso/caso.conf.sample

@@ -28,10 +28,15 @@
 # Service name within the site (string value)
 #service_name = $site_name
 
-# List of projects to extract accounting records from. (list value)
+# List of projects to extract accounting records from. You can use this option,
+# or add 'caso' tag to the project in Keystone. Please refer to the
+# documentation for more details. (list value)
 # Deprecated group/name - [DEFAULT]/tenants
 #projects =
 
+# Tag used to mark a project in Keystone to be extracted by cASO (string value)
+#caso_tag = caso
+
 # File containing the VO <-> project mapping as used in Keystone-VOMS. (string
 # value)
 #mapping_file = /etc/caso/voms.json

releasenotes/notes/tags-for-projects-3b9b6b5a92bcc6df.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Allow to use Keystone project tags to get usages from projects.

setup.cfg

@@ -46,6 +46,7 @@ oslo.config.opts =
 
 console_scripts =
     caso-extract = caso._cmd.extract:main
+    caso-projects = caso._cmd.projects:main
 
 caso.extractors =
     nova = caso.extract.openstack.nova:NovaExtractor