Fix dashboard functionalities (#84)

Author: Davide Schiera

examples/create_dashboard.py

@@ -29,7 +29,7 @@ def usage():
     usage()
 
 # Name for the dashboard to create
-dashboardName = "API test - cassandra in prod"
+dashboardName = "Overview by Process"
 for opt, arg in opts:
     if opt in ("-d", "--dashboard"):
         dashboardName = arg
@@ -55,8 +55,7 @@ def usage():
 # in Sysdig Cloud Explore page.
 # You can also refer to AWS tags by using "cloudProvider.tag.*" metadata or
 # agent tags by using "agent.tag.*" metadata
-dashboardFilter = "kubernetes.namespace.name = prod and proc.name = cassandra"
-
+dashboardFilter = "kubernetes.namespace.name = prod"
 print('Creating dashboard from view')
 ok, res = sdclient.create_dashboard_from_view(dashboardName, viewName, dashboardFilter)
 #
@@ -74,9 +73,9 @@ def usage():
 #
 
 # Name of the dashboard to copy
-dashboardCopy = "Copy Of {}".format(dashboardName)
+dashboardCopy = "Copy of {}".format(dashboardName)
 # Filter to apply to the new dashboard. Same as above.
-dashboardFilter = "kubernetes.namespace.name = dev and proc.name = cassandra"
+dashboardFilter = "kubernetes.namespace.name != prod"
 
 print('Creating dashboard from dashboard')
 ok, res = sdclient.create_dashboard_from_dashboard(dashboardCopy, dashboardName, dashboardFilter)

examples/dashboard_scope.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+#
+# This example shows some examples of scope you can use for dashboards.
+#
+
+import getopt
+import os
+import sys
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), '..'))
+from sdcclient import SdcClient
+
+
+#
+# Scopes can be passed to most of dashboard-related functions, e.g. create_dashboard_from_file.
+#
+# NOTE: convert_scope_string_to_expression should never be used in a user script
+# We're going to use it here just to demonstrate some scope options and some constraints
+#
+def evaluate(scope, expected):
+    parsed_scope = SdcClient.convert_scope_string_to_expression(scope)
+    print '{} is valid: {}'.format(scope, parsed_scope[0] == True)
+
+    if parsed_scope[0] != expected:
+        print('Unexpected parsing result!')
+        sys.exit(1)
+
+
+# simple example: tag = value
+evaluate('proc.name = "cassandra"', True)
+
+# NOTE: For now you can still leave values without quotes.
+# The API will be more strict, so please make sure you adopt the new format!
+evaluate('proc.name = cassandra', True)
+
+# other operators
+evaluate('proc.name != "cassandra"', True)
+evaluate('proc.name starts with "cassandra"', True)
+evaluate('proc.name contains "cassandra"', True)
+
+# list operators
+evaluate('proc.name in ("cassandra", "mysql")', True)
+
+# not-ed expressions
+evaluate('not proc.name starts with "cassandra"', True)
+evaluate('not proc.name contains "cassandra"', True)
+evaluate('not proc.name in ("cassandra", "mysql")', True)
+
+# you can combine multiple expressions; note that only AND'd scopes are currently supported
+evaluate('kubernetes.service.name = "database" and proc.name = "cassandra"', True)
+
+# the scope can obviously be omitted in the dashboard configuration
+evaluate('', True)
+evaluate(None, True)
+
+# invalid scopes will cause errors
+evaluate('proc.name == "cassandra"', False) # invalid operator
+
+# currently, one space is required around operands and operators -- improvements will come soon
+evaluate('proc.name="cassandra"', False)
+
+#
+# The current grammer is unable to validate all errors -- in these cases, the API will fail!
+# Improvements will come soon!
+#
+# Here some errors that will not be detected by the Python library, but the API will
+#
+evaluate('proc.name = "cassandra" or proc.name = "mysql"', True) # not AND'd expressions are supported
+evaluate('proc.name in ("cassandra\', \'mysql")', True) # mismatching quotes
+evaluate('proc.name in ("cassandra", "mysql"', True) # missing parenthesis

sdcclient/_monitor.py

@@ -1,6 +1,7 @@
 import json
 import copy
 import requests
+import re
 
 from sdcclient._common import _SdcCommon
 
@@ -430,31 +431,10 @@ def add_dashboard_panel(self, dashboard, name, panel_type, metrics, scope=None,
                 'groupAggregation': metric['aggregations']['group'] if 'aggregations' in metric else None,
                 'propertyName': property_name + str(i)
             })
-        #
-        # Convert scope to format used by Sysdig Monitor
-        #
-        if scope != None:
-            filter_expressions = scope.strip(' \t\n\r?!.').split(" and ")
-            filters = []
-
-            for filter_expression in filter_expressions:
-                values = filter_expression.strip(' \t\n\r?!.').split("=")
-                if len(values) != 2:
-                    return [False, "invalid scope format"]
-                filters.append({
-                    'metric': values[0].strip(' \t\n\r?!.'),
-                    'op': '=',
-                    'value': values[1].strip(' \t\n\r"?!.'),
-                    'filters': None
-                })
-
-            if len(filters) > 0:
-                panel_configuration['filter'] = {
-                    'filters': {
-                        'logic': 'and',
-                        'filters': filters
-                    }
-                }
+
+        panel_configuration['scope'] = scope
+        # if chart scope is equal to dashboard scope, set it as non override
+        panel_configuration['overrideFilter'] = ('scope' in dashboard and dashboard['scope'] != scope) or ('scope' not in dashboard and scope != None)
 
         #
         # Configure panel type
@@ -580,17 +560,32 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared
         template['isPublic'] = public
         template['publicToken'] = None
 
-        #
         # set dashboard scope to the specific parameter
-        # NOTE: Individual panels might override the dashboard scope, the override will NOT be reset
-        #
+        scopeExpression = self.convert_scope_string_to_expression(scope)
+        if scopeExpression[0] == False:
+            return scopeExpression
         template['filterExpression'] = scope
+        template['scopeExpressionList'] = map(lambda ex: {'operand':ex['operand'], 'operator':ex['operator'],'value':ex['value'],'displayName':'', 'isVariable':False}, scopeExpression[1])
+
+        if 'widgets' in template and template['widgets'] is not None:
+            # Default dashboards (aka Explore views) specify panels with the property `widgets`,
+            # while custom dashboards use `items`
+            template['items'] = list(template['widgets'])
+            del template['widgets']
 
-        if 'items' in template:
+        # NOTE: Individual panels might override the dashboard scope, the override will NOT be reset
+        if 'items' in template and template['items'] is not None:
             for chart in template['items']:
-                if 'overrideFilter' in chart and chart['overrideFilter'] == False:
+                if 'overrideFilter' not in chart:
+                    chart['overrideFilter'] = False
+
+                if chart['overrideFilter'] == False:
                     # patch frontend bug to hide scope override warning even when it's not really overridden
                     chart['scope'] = scope
+                
+                # if chart scope is equal to dashboard scope, set it as non override
+                chart_scope = chart['scope'] if 'scope' in chart else None
+                chart['overrideFilter'] = chart_scope != scope
 
         if 'annotations' in template:
             template['annotations'].update(annotations)
@@ -754,6 +749,64 @@ def get_metrics(self):
         res = requests.get(self.url + '/api/data/metrics', headers=self.hdrs, verify=self.ssl_verify)
         return self._request_result(res)
 
+    @staticmethod
+    def convert_scope_string_to_expression(scope):
+        '''**Description**
+            Internal function to convert a filter string to a filter object to be used with dashboards.
+        '''
+        #
+        # NOTE: The supported grammar is not perfectly aligned with the grammar supported by the Sysdig backend.
+        # Proper grammar implementation will happen soon.
+        # For practical purposes, the parsing will have equivalent results.
+        #
+
+        if scope is None or not scope:
+            return [True, []]
+
+        expressions = []
+        string_expressions = scope.strip(' \t\n\r').split(' and ')
+        expression_re = re.compile('^(?P<not>not )?(?P<operand>[^ ]+) (?P<operator>=|!=|in|contains|starts with) (?P<value>(:?"[^"]+"|\'[^\']+\'|\(.+\)|.+))$')
+
+        for string_expression in string_expressions:
+            matches = expression_re.match(string_expression)
+
+            if matches is None:
+                return [False, 'invalid scope format']
+
+            is_not_operator = matches.group('not') is not None
+
+            if matches.group('operator') == 'in':
+                list_value = matches.group('value').strip(' ()')
+                value_matches = re.findall('(:?\'[^\',]+\')|(:?"[^",]+")|(:?[,]+)', list_value)
+
+                if len(value_matches) == 0:
+                    return [False, 'invalid scope value list format']
+
+                value_matches = map(lambda v: v[0] if v[0] else v[1], value_matches)
+                values = map(lambda v: v.strip(' "\''), value_matches)
+            else:
+                values = [matches.group('value').strip('"\'')]
+
+            operator_parse_dict = {
+                'in': 'in' if not is_not_operator else 'notIn',
+                '=': 'equals' if not is_not_operator else 'notEquals',
+                '!=': 'notEquals' if not is_not_operator else 'equals',
+                'contains': 'contains' if not is_not_operator else 'notContains',
+                'starts with': 'startsWith'
+            }
+
+            operator = operator_parse_dict.get(matches.group('operator'), None)
+            if operator is None:
+                return [False, 'invalid scope operator']
+
+            expressions.append({
+                'operand': matches.group('operand'),
+                'operator': operator,
+                'value': values
+            })
+
+        return [True, expressions]
+
 
 # For backwards compatibility
 SdcClient = SdMonitorClient

test/test_monitor_apis.sh

@@ -21,6 +21,7 @@ date; $SCRIPTDIR/../examples/create_alert.py -a $ALERT_NAME $PYTHON_SDC_TEST_MON
 date; $SCRIPTDIR/../examples/update_alert.py -a $ALERT_NAME $PYTHON_SDC_TEST_MONITOR_API_TOKEN
 date; $SCRIPTDIR/../examples/delete_alert.py -a $ALERT_NAME $PYTHON_SDC_TEST_MONITOR_API_TOKEN
 date; $SCRIPTDIR/../examples/dashboard.py -d $DASHBOARD_1_NAME $PYTHON_SDC_TEST_MONITOR_API_TOKEN
+date; $SCRIPTDIR/../examples/dashboard_scope.py
 date; $SCRIPTDIR/../examples/create_dashboard.py -d $DASHBOARD_2_NAME $PYTHON_SDC_TEST_MONITOR_API_TOKEN
 date; $SCRIPTDIR/../examples/delete_dashboard.py -p $SESSION_UUID $PYTHON_SDC_TEST_MONITOR_API_TOKEN
 date; $SCRIPTDIR/../examples/get_data_advanced.py $PYTHON_SDC_TEST_MONITOR_API_TOKEN $AGENT_HOSTNAME