diff --git a/checks.d/docker_daemon.py b/checks.d/docker_daemon.py index 799d1e9fb4..17deb09552 100644 --- a/checks.d/docker_daemon.py +++ b/checks.d/docker_daemon.py @@ -643,56 +643,78 @@ def _pre_aggregate_events(self, api_events, containers_by_id): def _format_events(self, aggregated_events, containers_by_id): events = [] for image_name, event_group in aggregated_events.iteritems(): - max_timestamp = 0 - status = defaultdict(int) - status_change = [] container_tags = set() + low_prio_events = [] + normal_prio_events = [] + for event in event_group: - max_timestamp = max(max_timestamp, int(event['time'])) - status[event['status']] += 1 container_name = event['id'][:11] + if event['id'] in containers_by_id: cont = containers_by_id[event['id']] container_name = DockerUtil.container_name_extractor(cont)[0] container_tags.update(self._get_tags(cont, PERFORMANCE)) container_tags.add('container_name:%s' % container_name) - status_change.append([container_name, event['status']]) - - status_text = ", ".join(["%d %s" % (count, st) for st, count in status.iteritems()]) - msg_title = "%s %s on %s" % (image_name, status_text, self.hostname) - msg_body = ( - "%%%\n" - "{image_name} {status} on {hostname}\n" - "```\n{status_changes}\n```\n" - "%%%" - ).format( - image_name=image_name, - status=status_text, - hostname=self.hostname, - status_changes="\n".join( - ["%s \t%s" % (change[1].upper(), change[0]) for change in status_change]) - ) - - if any(error in status_text for error in ERROR_ALERT_TYPE): - alert_type = "error" - else: - alert_type = None - - events.append({ - 'timestamp': max_timestamp, - 'host': self.hostname, - 'event_type': EVENT_TYPE, - 'msg_title': msg_title, - 'msg_text': msg_body, - 'source_type_name': EVENT_TYPE, - 'event_object': 'docker:%s' % image_name, - 'tags': list(container_tags), - 'alert_type': alert_type - }) + # health checks generate tons of these so we treat them separately and lower their priority + if event['status'].startswith('exec_create:') or event['status'].startswith('exec_start:'): + low_prio_events.append((event, container_name)) + else: + normal_prio_events.append((event, container_name)) + + exec_event = self._create_dd_event(low_prio_events, image_name, container_tags, priority='Low') + events.append(exec_event) + + normal_event = self._create_dd_event(normal_prio_events, image_name, container_tags, priority='Normal') + events.append(normal_event) return events + def _create_dd_event(self, events, image, c_tags, priority='Normal'): + """Create the actual event to submit from a list of similar docker events""" + max_timestamp = 0 + status = defaultdict(int) + status_change = [] + + for ev, c_name in events: + max_timestamp = max(max_timestamp, int(ev['time'])) + status[ev['status']] += 1 + status_change.append([c_name, ev['status']]) + + status_text = ", ".join(["%d %s" % (count, st) for st, count in status.iteritems()]) + msg_title = "%s %s on %s" % (image, status_text, self.hostname) + msg_body = ( + "%%%\n" + "{image_name} {status} on {hostname}\n" + "```\n{status_changes}\n```\n" + "%%%" + ).format( + image_name=image, + status=status_text, + hostname=self.hostname, + status_changes="\n".join( + ["%s \t%s" % (change[1].upper(), change[0]) for change in status_change]) + ) + + if any(error in status_text for error in ERROR_ALERT_TYPE): + alert_type = "error" + else: + alert_type = None + + return { + 'timestamp': max_timestamp, + 'host': self.hostname, + 'event_type': EVENT_TYPE, + 'msg_title': msg_title, + 'msg_text': msg_body, + 'source_type_name': EVENT_TYPE, + 'event_object': 'docker:%s' % image, + 'tags': list(c_tags), + 'alert_type': alert_type, + 'priority': priority + } + + def _report_disk_stats(self): """Report metrics about the volume space usage""" stats = {