Merge pull request #5 from stackhpc/placement

Add Prometheus exporter for usage

Author: JohnGarbutt

.gitignore

@@ -45,6 +45,7 @@ nosetests.xml
 coverage.xml
 *.cover
 .hypothesis/
+.stestr
 
 # Translations
 *.mo

.stestr.conf

@@ -0,0 +1,3 @@
+[DEFAULT]
+test_path=./os_capacity/tests
+top_dir=./

README.rst

@@ -135,23 +135,8 @@ See the online help for more details:
       complete       print bash completion command
       flavor list    List all the flavors.
       help           print detailed help for another command
+      prometheus  To be run as node exporter textfile collector
       resources all  List all resource providers, with their resources and servers.
       resources group  Lists counts of resource providers with similar inventories.
       usages all     List all current resource usages.
       usages group   Group usage by specified key (by user or project).
-
-Submitting Metrics to Monasca
------------------------------
-
-There is now an experimental mode where metrics can be written into Monasca
-for the calls "resources group" and "usages group user". To enable this
-feature you must set::
-
-    export OS_CAPACITY_SEND_METRICS=1
-
-To later disable the feature you must unset the environment variable::
-
-    unset OS_CAPACITY_SEND_METRICS
-
-For an example of using this with cron, please see the example script in
-``cron/example.sh``.

os_capacity/commands/commands.py

@@ -19,6 +19,15 @@
 
 from os_capacity.data import metrics
 from os_capacity import utils
+from os_capacity.data import candidates
+
+
+class PrometheusAll(Lister):
+    """To be run as node exporter textfile collector."""
+    def take_action(self, parsed_args):
+        candidates.print_exporter_data(self.app)
+        # TODO(johngarbutt) a total hack!
+        return (('fake'), [])
 
 
 class FlavorList(Lister):

os_capacity/data/candidates.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+
+import collections
+import json
+
+import openstack
+
+
+def get_capacity_per_flavor(placement_client, flavors):
+    capacity_per_flavor = {}
+
+    for flavor in flavors:
+        resources, traits = get_placement_request(flavor)
+        max_per_host = get_max_per_host(placement_client, resources, traits)
+        capacity_per_flavor[flavor.name] = max_per_host
+
+    return capacity_per_flavor
+
+
+def get_placement_request(flavor):
+    resources = {"MEMORY_MB": flavor.ram, "DISK_GB": (flavor.disk + flavor.ephemeral)}
+    required_traits = []
+    for key, value in flavor.extra_specs.items():
+        if "trait:" == key[:6]:
+            if value == "required":
+                required_traits.append(key[6:])
+        if "resources:" == key[:10]:
+            count = int(value)
+            resources[key[10:]] = count
+        if "hw:cpu_policy" == key and value == "dedicated":
+            resources["PCPU"] = flavor.vcpus
+    if "PCPU" not in resources.keys() and "VCPU" not in resources.keys():
+        resources["VCPU"] = flavor.vcpus
+    return resources, required_traits
+
+
+def get_max_per_host(placement_client, resources, required_traits):
+    resource_str = ",".join(
+        [key + ":" + str(value) for key, value in resources.items() if value]
+    )
+    required_str = ",".join(required_traits)
+    # TODO(johngarbut): remove disabled!
+    forbidden_str = "COMPUTE_STATUS_DISABLED"
+
+    response = placement_client.get(
+        "/allocation_candidates",
+        params={"resources": resource_str, "required": required_str},
+        headers={"OpenStack-API-Version": "placement 1.29"},
+    )
+    raw_data = response.json()
+    count_per_rp = {}
+    for rp_uuid, summary in raw_data.get("provider_summaries", {}).items():
+        # per resource, get max possible number of instances
+        max_counts = []
+        for resource, amounts in summary["resources"].items():
+            requested = resources.get(resource, 0)
+            if requested:
+                free = amounts["capacity"] - amounts["used"]
+                amount = int(free / requested)
+                max_counts.append(amount)
+        # available count is the min of the max counts
+        if max_counts:
+            count_per_rp[rp_uuid] = min(max_counts)
+
+    return count_per_rp
+
+
+def print_exporter_data(app):
+    flavors = list(app.compute_client.flavors())
+    capacity_per_flavor = get_capacity_per_flavor(app.placement_client, flavors)
+
+    # total capacity per flavor
+    flavor_names = sorted([f.name for f in flavors])
+    for flavor_name in flavor_names:
+        counts = capacity_per_flavor.get(flavor_name, {}).values()
+        total = 0 if not counts else sum(counts)
+        print(f'openstack_total_capacity_per_flavor{{flavor="{flavor_name}"}} {total}')
+
+    # capacity per host
+    raw_rps = list(app.placement_client.resource_providers())
+    resource_providers = {rp.name: rp.id for rp in raw_rps}
+    hostnames = sorted(resource_providers.keys())
+    for hostname in hostnames:
+        rp_id = resource_providers[hostname]
+        for flavor_name in flavor_names:
+            all_counts = capacity_per_flavor.get(flavor_name, {})
+            our_count = all_counts.get(rp_id, 0)
+            if our_count == 0:
+                continue
+            print(
+                f'openstack_capacity_by_hostname{{hypervisor="{hostname}",flavor="{flavor_name}"}} {our_count}'
+            )

os_capacity/shell.py

@@ -18,6 +18,7 @@
 from cliff.commandmanager import CommandManager
 import os_client_config
 
+import openstack
 
 def get_cloud_config():
     # TODO(johngarbutt) consider passing in argument parser
@@ -41,11 +42,11 @@ def __init__(self):
     def initialize_app(self, argv):
         self.LOG.debug('initialize_app')
 
-        config = os_client_config.get_config()
-        self.compute_client = config.get_session_client("compute")
-        self.placement_client = config.get_session_client("placement")
-        self.monitoring_client = config.get_session_client("monitoring")
-        self.identity_client = config.get_session_client("identity")
+        conn = openstack.openstack.connect()
+        self.connection = conn
+        self.compute_client = conn.compute
+        self.placement_client = conn.placement
+        self.identity_client = conn.identity
 
         self.LOG.debug('setup Keystone API REST clients')
 

os_capacity/tests/unit/test_data.py

@@ -13,8 +13,8 @@
 # under the License.
 
 import datetime
-import mock
 import unittest
+from unittest import mock
 
 from os_capacity.data import flavors
 from os_capacity.data import resource_provider

os_capacity/tests/unit/test_utils.py

@@ -13,8 +13,8 @@
 # under the License.
 
 import datetime
-import mock
 import unittest
+from unittest import mock
 
 from os_capacity.data import flavors
 from os_capacity.data import resource_provider
@@ -26,15 +26,18 @@ class TestUtils(unittest.TestCase):
 
     @mock.patch.object(flavors, "get_all")
     def test_get_flavors(self, mock_flavors):
-        mock_flavors.return_value = [flavors.Flavor(
-            id="id", name="name", vcpus=1, ram_mb=2, disk_gb=3)]
+        mock_list = [flavors.Flavor(
+            id="id", name="name", vcpus=1, ram_mb=2, disk_gb=3,
+            extra_specs={})]
+        mock_flavors.return_value = mock_list
         app = mock.Mock()
 
         result = utils.get_flavors(app)
 
-        mock_flavors.assert_called_once_with(app.compute_client)
-        expected_flavors = [('id', 'name', 1, 2, 3)]
-        self.assertEqual(expected_flavors, result)
+        mock_flavors.assert_called_once_with(
+            app.compute_client, include_extra_specs=True)
+        expected_flavors = [('id', 'name', 1, 2, 3, {})]
+        self.assertEqual(mock_list, result)
 
     @mock.patch.object(resource_provider, 'get_allocations')
     @mock.patch.object(resource_provider, 'get_inventories')

setup.py

@@ -57,6 +57,7 @@
             '.commands:ListResourcesGroups',
             'usages_all = os_capacity.commands.commands:ListUsagesAll',
             'usages_group = os_capacity.commands.commands:ListUsagesGroup',
+            'prometheus = os_capacity.commands.commands:PrometheusAll',
         ],
     },
 )

test-requirements.txt

@@ -1,10 +1,11 @@
 # The order of packages is significant, because pip processes them in the order
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
+hacking>=3.0,<3.1 # Apache-2.0
 
-hacking>=0.12.0,<0.13 # Apache-2.0
-
-coverage>=4.0 # Apache-2.0
-doc8 # Apache-2.0
-sphinx>=1.5.1 # BSD
+black
+coverage>=4.0,!=4.4 # Apache-2.0
+python-subunit>=0.0.18 # Apache-2.0/BSD
 oslotest>=1.10.0 # Apache-2.0
+stestr>=1.0.0 # Apache-2.0
+testtools>=1.4.0 # MIT