Merge pull request #51 from ARGOeu/devel

Version 1.0.0

Author: themiszamani

CHANGELOG.md

@@ -0,0 +1,58 @@
+# Changelog
+
+All notable changes in argo-alert project are documented here
+
+## 1.0.0 - (2023-02-02)
+
+### Added:
+- ARGO-3901 Retrieve contacts from argo-web-api
+
+### Changed:
+- ARGO-2856 Update simple feed parameters for json and csv
+- ARGO-2856 Update notifications to read contact info from json topology
+
+
+## 0.2.1 - (2020-12-16)
+
+### Added:
+- ARGO-2079 Add original contacts feature when sending to testing emails
+
+### Changed:
+- ARGO-2145 Finalize py3 version in argo-alert scripts
+
+### Fixed:
+- ARGO-2027 Split gocdb contain email string into individual items
+
+
+## 0.2.0 - (2019-11-06)
+
+### Added:
+- ARGO-2027 Split gocdb contain email string into individual items
+- ARGO-1710 publish group item status info
+- ARGO-1715 Consolidate alerts for endpoints that belong to multiple endpoint groups
+- ARGO-1640 Update alert publisher to forward new event information
+
+### Fixed:
+- ARGO-1793 Fix ui_urls in alert mails to point correctly to the new web_ui
+
+
+## 0.1.2 - (2018-11-09)
+
+### Fixed:
+- ARGO-1464 Update requests dep to 2.20
+
+
+## 0.1.1 - (2018-02-27) 
+
+### Added:
+- ARGO-933 Implement argo-alert publisher of argo-streaming events to alerta
+- ARGO-931 Alerta-mailer rule generator from gocdb contact data
+- ARGO-943 Use requests for verifying cert using capath (directory).
+- ARGO 960 Refactor rule generator for using notifications tag
+- ARGO-990 Accept a list of kafka endpoints for publisher
+- ARGO-999 Add boolean conf parameter for using contact notifications
+- ARGO-1026 Support different levels of entity groups when retrieving
+- ARGO-996 Add ability to generate rules using a group of test email
+
+### Fixed:
+- ARGO-945 Fix endpoint_group field. Lowercase status field values

Jenkinsfile

@@ -10,7 +10,7 @@ pipeline {
         stage ('Test Centos 7') {
             agent {
                 docker {
-                    image 'argo.registry:5000/epel-7-ams'
+                    image 'argo.registry:5000/python3'
                     args '-u jenkins:jenkins'
                 }
             }
@@ -19,7 +19,7 @@ pipeline {
                     cd ${WORKSPACE}/$PROJECT_DIR
                     pipenv install -r requirements.txt
                     pipenv run python setup.py install
-                    pipenv run coverage run --source=./argoalert -m py.test
+                    pipenv run coverage run --source=./argoalert -m pytest
                     pipenv run coverage xml
                     pipenv run py.test ./ --junitxml=./junit.xml
                 '''

README.md

@@ -107,6 +107,10 @@ extra-emails=alert01@mail.example.foo,alert02@mail.example.foo
 aleart-timeout = 3600
 # group type of the tenant's top level group used in alert generation and mail template
 group-type = Group
+# endpoint type could be set to the default: endpoint or to another type (for example: service instance)
+endpoint-type = endpoint
+# ui-path-group is used in argo ui's result to define the correct type of group in results path
+ui-path-group = SITES
 # report name used in argo-web-ui url construction
 report = Critical
 

argoalert/argoalert.py

@@ -37,7 +37,7 @@ def date_end_of_day(dt):
     return dt.replace(hour=23, minute=59, second=59)
 
 
-def ui_group_url(ui_endpoint, report, timestamp, grouptype, group,  environment):
+def ui_group_url(ui_endpoint, report, timestamp, grouptype, group, environment, ui_path_group):
     """Generate an http url to a relevant argo web ui endpoint group timeline page
 
             Args:
@@ -54,10 +54,10 @@ def ui_group_url(ui_endpoint, report, timestamp, grouptype, group,  environment)
     start_date = date_only_string(date_days_ago(parse_timestamp(timestamp), 3))
     end_date = date_only_string(parse_timestamp(timestamp))
     return "https://{0}/{1}/report-status/{2}/{3}/{4}?start={5}&end={6}".format(
-        ui_endpoint, environment.lower(), report, grouptype.upper() + "S", group, start_date, end_date)
+        ui_endpoint, environment.lower(), report, ui_path_group, group, start_date, end_date)
 
 
-def ui_endpoint_url(ui_endpoint, report, timestamp, grouptype, group, service, hostname, environment):
+def ui_endpoint_url(ui_endpoint, report, timestamp, grouptype, group, service, hostname, environment, ui_path_group):
     """Generate an http url to a relevant argo web ui endpoint timeline page
 
             Args:
@@ -76,10 +76,10 @@ def ui_endpoint_url(ui_endpoint, report, timestamp, grouptype, group, service, h
     start_date = date_only_string(date_days_ago(parse_timestamp(timestamp), 3))
     end_date = date_only_string(parse_timestamp(timestamp))
     return "http://{0}/{1}/report-status/{2}/{3}/{4}/{5}/{6}?start={7}&end={8}".format(
-        ui_endpoint, environment.lower(), report, grouptype.upper() + "S", group, service, hostname,  start_date, end_date)
+        ui_endpoint, environment.lower(), report, ui_path_group, group, service, hostname,  start_date, end_date)
 
 
-def transform(argo_event, environment, grouptype, timeout, ui_endpoint, report):
+def transform(argo_event, environment, grouptype, timeout, ui_endpoint, report, endpoint_type, ui_path_group):
     """Transform an argo status event to an alerta alert
 
     Args:
@@ -181,7 +181,7 @@ def transform(argo_event, environment, grouptype, timeout, ui_endpoint, report):
             environment.upper(), grouptype.capitalize(), group, status.upper())
         if ui_endpoint != "":
             attributes["_alert_url"] = ui_group_url(
-                ui_endpoint, report, ts_monitored, grouptype, group, environment)
+                ui_endpoint, report, ts_monitored, grouptype, group, environment, ui_path_group)
 
     elif etype == "service":
         alerta_service.append("service")
@@ -192,11 +192,11 @@ def transform(argo_event, environment, grouptype, timeout, ui_endpoint, report):
     elif etype == "endpoint":
         alerta_service.append("endpoint")
         resource = service + "/" + hostname
-        text = "[ {0} ] - Endpoint {1}/{2} is {3}".format(
-            environment.upper(), hostname, service, status.upper())
+        text = "[ {0} ] - {4} {1} ({2}) is {3}".format(
+                environment.upper(), hostname, service, status.upper(), endpoint_type.capitalize())
         if ui_endpoint != "":
             attributes["_alert_url"] = ui_endpoint_url(
-                ui_endpoint, report, ts_monitored, grouptype, group, service, hostname, environment)
+                ui_endpoint, report, ts_monitored, grouptype, group, service, hostname, environment, ui_path_group)
 
     elif etype == "metric":
         alerta_service.append("metric")
@@ -235,7 +235,7 @@ def read_and_send(message, environment, alerta_url, alerta_token, options):
 
     try:
         alerta = transform(argo_event, environment,
-                           options["group_type"], options["timeout"], options["ui_endpoint"], options["report"])
+                           options["group_type"], options["timeout"], options["ui_endpoint"], options["report"], options["endpoint_type"], options["ui_path_group"])
     except KeyError as e:
         logging.warning("WRONG JSON SCHEMA: " + message.value)
         return
@@ -341,26 +341,53 @@ def gocdb_to_contacts(gocdb_xml, use_notif_flag, test_emails):
                 name = name_tags[0].firstChild.nodeValue
                 service = service_tags[0].firstChild.nodeValue
                 c["name"] = "\\/" + service + "\\/" + name
-            
+
             if test_emails is None:
                 c["emails"] = item.firstChild.nodeValue
             else:
                 c["emails"] = test_emails[indx % len(test_emails)]
                 c["original_email"] = item.firstChild.nodeValue
                 indx = indx + 1
-            
+
             contacts.append(c)
 
     return contacts
 
 
+def gen_endpoint_contacts_from_groups(group_data, endpoint_data, group_filter=None):
+    """Parse both endpoint and group topology data and refactor endpoint data using
+       notification information from groups
+
+    Args:
+        endpoint_data (obj): list of endpoint topology data
+        group_data (string): list of group topology data
+        group_filter (string): keep only groups of a specific type e.g. SITES
+
+    Returns:
+        obj: list containing endpoint topology data with notification detailes borrowed from groups
+    """
+    group_index = {}
+    gen_endpoints = []
+    for item in group_data:
+            group_index[item["subgroup"]] = item["notifications"]
+
+    for item in endpoint_data:
+        endpoint_group = item["group"]
+        if group_filter and group_filter != item["type"]:
+            # skip endpoint if we have enabled a group filter and its group type doesn't match
+            continue
+        if endpoint_group in group_index:
+            item["notifications"] = group_index[endpoint_group]
+            gen_endpoints.append(item)
+
+    return gen_endpoints
 
 def argo_web_api_to_contacts(endpoint_data, group_data, use_notif=False, test_emails=None):
     """Contact argo-web-api endpoint and retrieve contacts for topology endpoints and groups
 
     Args:
-        api_endpoint (string): endpoint for argo-web-api instance
-        access_key (string): argo-web-api access key
+        endpoint_data (obj): list of endpoint topology data
+        grou_data (string): list of group topology data
         verify (bool, optional): Set https verification on/off. Defaults to True.
         test_emails (string, optional): Set test emails for notifications. Defaults to None
 
@@ -371,40 +398,61 @@ def argo_web_api_to_contacts(endpoint_data, group_data, use_notif=False, test_em
     get_notif_always = not use_notif
 
 
-    
+
     contacts = []
     subgroup_types = {}
     # iterate over endpoints but also get subgroup types
     for indx, endpoint in enumerate(endpoint_data):
         subgroup_types[endpoint["group"]]=endpoint["type"]
         if "notifications" in endpoint:
-            if get_notif_always or (endpoint["notifications"]["enabled"] == True):
-                name = "{}\\/{}".format(endpoint["service"].replace(".","\\."),endpoint["hostname"].replace(".","\\."))
+            # if notifications is empty
+            if get_notif_always or ("enabled" in endpoint["notifications"] and endpoint["notifications"]["enabled"] == True):
+                name = "{}\\/{}".format(endpoint["service"].replace(".","\\."), endpoint["hostname"].replace(".","\\."))
+                contact = ""
                 if not test_emails:
-                    contact = ";".join(endpoint["notifications"]["contacts"])
+                    if "contacts" in endpoint["notifications"]:
+                        contact = ";".join(endpoint["notifications"]["contacts"])
                 else:
-                    contact  = test_emails[indx % len(test_emails)]
-                contacts.append({"name":name, "emails":contact, "type":endpoint["type"]})
+                    contact = test_emails[indx % len(test_emails)]
+                if contact != "":
+                    contacts.append({"name": name, "emails": contact, "type": "endpoint"})
+
+    # consolidate rules with duplicate names (same endpoint belonging to multiple groups)
+    contact_index = {}
+    for contact in contacts:
+        if contact["name"] in contact_index:
+            # append emails to existing contact rule
+            contact_index[contact["name"]]["emails"] = contact_index[contact["name"]]["emails"] + ";" + contact["emails"]
+        else:
+            contact_index[contact["name"]] = contact
+
+    contacts = list(contact_index.values())
+
     # iterate now over group data
     for indx, group in enumerate(group_data):
         if "notifications" not in group:
             continue
-        if get_notif_always or (group["notifications"]["enabled"] == True):
+
+        if get_notif_always or ("enabled" in group["notifications"] and group["notifications"]["enabled"] == True):
             name = group["subgroup"]
             if name not in subgroup_types:
                 continue
             subgroup_type = subgroup_types[name]
+            contact = ""
             if not test_emails:
-                contact = ";".join(endpoint["notifications"]["contacts"])
+                if "contacts" in group["notifications"]:
+                    contact = ";".join(group["notifications"]["contacts"])
             else:
                 contact  = test_emails[indx % len(test_emails)]
-            contacts.append({"name":name, "emails":contact, "type":subgroup_type})
+            if contact != "":
+                contacts.append({"name":name, "emails":contact, "type":subgroup_type})
+
     return contacts
 
 
 def get_argo_web_api_data(api_endpoint, access_key, verify=True, topology_type="endpoints"):
      # get endpoint topology
-    
+
     api_url = api_endpoint
     if topology_type == "endpoints":
         api_url = api_url + "/api/v2/topology/endpoints"
@@ -414,7 +462,7 @@ def get_argo_web_api_data(api_endpoint, access_key, verify=True, topology_type="
     logging.info("Requesting topology of {}  from argo-web-api: {}".format(topology_type,api_url))
     r = requests.get(api_url, headers={
                      'x-api-key': access_key, 'Accept': 'application/json'}, verify=verify)
-    
+
     if r.status_code == 200:
         logging.info("Argo-web-api topology data retrieved successfully")
         return json.loads(r.text)["data"]
@@ -480,15 +528,15 @@ def contacts_to_alerta(contacts, extras=None, environment=None):
             rule_fields.append(
                 {"field": "environment", "regex": "{0}".format(environment)})
         rule_contacts = re.split(";|,", c["emails"].replace(" ", ""))
-        
+
         if extras:
             rule_contacts.extend(extras)
         rule_exclude = True
 
         rule = {"name": rule_name, "fields": rule_fields,
                 "contacts": rule_contacts, "exclude": rule_exclude}
-        
-        
+
+
 
         # Check if contacts have original emails -- used during testing
         if "original_email" in c:
@@ -536,10 +584,6 @@ def get_gocdb(api_url, auth_info, ca_bundle):
 
     return ""
 
-
-
-
-
 def write_rules(rules, outfile):
     """Writes alerta email rules to a specific output file
 
@@ -551,4 +595,4 @@ def write_rules(rules, outfile):
     json_str = json.dumps(rules, indent=4)
     logging.info("Saving rule to file: " + outfile)
     with open(outfile, "w") as output_file:
-        output_file.write(json_str)
+        output_file.write(json_str)

bin/argo-alert-publisher

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/opt/alerta/bin/python
 
 from argparse import ArgumentParser
 from configparser import ConfigParser
@@ -19,7 +19,7 @@ def main(args=None):
     environment = parser.get("alerta", "environment")
     alerta_token = parser.get("alerta", "token")
     log_level = parser.get("logging", "level")
-    logging.basicConfig(level=log_level)
+    logging.basicConfig(level=log_level,format='%(asctime)s - %(levelname)s - %(message)s')
 
     # default alert timeout value
     alert_timeout = 3600
@@ -30,6 +30,15 @@ def main(args=None):
     if parser.has_option("alerta", "group-type"):
         group_type = parser.get("alerta", "group-type")
 
+    ui_path_group = "SITES"
+    if parser.has_option("alerta", "ui-path-group"):
+        ui_path_group = parser.get("alerta", "ui-path-group")
+
+
+    endpoint_type = "Endpoint"
+    if parser.has_option("alerta", "endpoint-type"):
+        endpoint_type = parser.get("alerta", "endpoint-type")
+
     report = "Critical"
     if parser.has_option("alerta", "report"):
         report = parser.get("alerta", "report")
@@ -39,8 +48,10 @@ def main(args=None):
         ui_endpoint = parser.get("alerta", "ui_endpoint")
 
     # create alert options dictionary
-    options = {"timeout": alert_timeout, "group_type": group_type,
-               "report": report, "ui_endpoint": ui_endpoint}
+    options = {"timeout": alert_timeout, "group_type": group_type, "endpoint_type": endpoint_type,
+            "report": report, "ui_endpoint": ui_endpoint, "ui_path_group":ui_path_group }
+
+
 
     argoalert.start_listening(environment, kafka_endpoint,
                               kafka_topic, alerta_endpoint, alerta_token, options)
@@ -56,4 +67,4 @@ if __name__ == "__main__":
 
     # Parse the command line arguments accordingly and introduce them to
     # main...
-    sys.exit(main(arg_parser.parse_args()))
+    sys.exit(main(arg_parser.parse_args()))