[docker_daemon] adding healthcheck service checks.

[docker_daemon] set proper tagging and SC name for healthcheck.

[docker_daemon] put healtcheck sc behind feature flag - it requires inspect.

[ci] fixing docker_daemon CI.

[ci] fixing old setuptools setup_env issues. Bumping to newest version.

[docker_daemon] adding test for container inspeciton logic.

[docker_daemon] amending config yaml.

Author: truthbk

checks.d/docker_daemon.py

@@ -23,6 +23,7 @@
 
 EVENT_TYPE = 'docker'
 SERVICE_CHECK_NAME = 'docker.service_up'
+HEALTHCHECK_SERVICE_CHECK_NAME = 'docker.container_health'
 SIZE_REFRESH_RATE = 5  # Collect container sizes every 5 iterations of the check
 MAX_CGROUP_LISTING_RETRIES = 3
 CONTAINER_ID_RE = re.compile('[0-9a-f]{64}')
@@ -250,8 +251,11 @@ def check(self, instance):
         # containers running with custom cgroups?
         custom_cgroups = _is_affirmative(instance.get('custom_cgroups', False))
 
+        # submit healtcheck service checks?
+        health_service_checks = _is_affirmative(instance.get('health_service_checks', False))
+
         # Get the list of containers and the index of their names
-        containers_by_id = self._get_and_count_containers(custom_cgroups)
+        containers_by_id = self._get_and_count_containers(custom_cgroups, health_service_checks)
         containers_by_id = self._crawl_container_pids(containers_by_id, custom_cgroups)
 
         # Send events from Docker API
@@ -268,6 +272,10 @@ def check(self, instance):
         if self.collect_disk_stats:
             self._report_disk_stats()
 
+        # Report docker healthcheck SC's where available
+        if health_service_checks:
+            self._send_container_healthcheck_sc(containers_by_id)
+
     def _count_and_weigh_images(self):
         try:
             tags = self._get_tags()
@@ -284,7 +292,7 @@ def _count_and_weigh_images(self):
             # It's not an important metric, keep going if it fails
             self.warning("Failed to count Docker images. Exception: {0}".format(e))
 
-    def _get_and_count_containers(self, custom_cgroups=False):
+    def _get_and_count_containers(self, custom_cgroups=False, healthchecks=False):
         """List all the containers from the API, filter and count them."""
 
         # Querying the size of containers is slow, we don't do it at each run
@@ -328,10 +336,11 @@ def _get_and_count_containers(self, custom_cgroups=False):
 
             # grab pid via API if custom cgroups - otherwise we won't find process when
             # crawling for pids.
-            if custom_cgroups:
+            if custom_cgroups or healthchecks:
                 try:
                     inspect_dict = self.docker_client.inspect_container(container_name)
                     container['_pid'] = inspect_dict['State']['Pid']
+                    container['health'] = inspect_dict['State'].get('Health', {})
                 except Exception as e:
                     self.log.debug("Unable to inspect Docker container: %s", e)
 
@@ -512,6 +521,20 @@ def _report_container_size(self, containers_by_id):
                     self, 'docker.container.size_rootfs', container['SizeRootFs'],
                     tags=tags)
 
+    def _send_container_healthcheck_sc(self, containers_by_id):
+        for container in containers_by_id.itervalues():
+            health = container.get('health', {})
+            tags = self._get_tags(container, CONTAINER)
+            status = AgentCheck.UNKNOWN
+            if health:
+                _health = health.get('Status', '')
+                if _health == 'unhealthy':
+                    status = AgentCheck.CRITICAL
+                elif _health == 'healthy':
+                    status = AgentCheck.OK
+
+            self.service_check(HEALTHCHECK_SERVICE_CHECK_NAME, status, tags=tags)
+
     def _report_image_size(self, images):
         for image in images:
             tags = self._get_tags(image, IMAGE)

ci/docker_daemon.rb

@@ -8,6 +8,8 @@
   namespace :docker_daemon do |flavor|
     task before_install: ['ci:common:before_install']
 
+    task install: ['ci:common:install']
+
     task before_script: ['ci:common:before_script']
 
     task script: ['ci:common:script'] do

conf.d/docker_daemon.yaml.example

@@ -47,10 +47,18 @@ instances:
 
     # Do you use custom cgroups for this particular instance?
     # Note: enabling this option modifies the way in which we inspect the containers and causes
-    #       some overhead - if you run a high volume of containers we may timeout. Please only
-    #       enable if absolutely necessary.
+    #       some overhead - if you run a high volume of containers we may timeout.
+    #
     # custom_cgroups: false
 
+    # Report docker container healthcheck events as service checks
+    # Note: enabling this option modifies the way in which we inspect the containers and causes
+    #       some overhead - if you run a high volume of containers we may timeout.
+    #       Container Healthchecks are available starting with docker 1.12, enabling with older
+    #       versions will result in an UNKNOWN state for the service check.
+    #
+    # health_service_checks: false
+
     # Collect images stats
     # Number of available active images and intermediate images as gauges.
     # Defaults to false.

tests/checks/integration/test_docker_daemon.py

@@ -529,11 +529,29 @@ def test_events(self):
             },
             ],
         }
+
         DockerUtil().set_docker_settings(config['init_config'], config['instances'][0])
 
         self.run_check(config, force_reload=True)
         self.assertEqual(len(self.events), 2)
 
+    def test_healthcheck(self):
+        config = {
+            "init_config": {},
+            "instances": [{
+                "url": "unix://var/run/docker.sock",
+                "collect_images_stats": True,
+                "health_service_checks": True,
+            },
+            ],
+        }
+
+        DockerUtil().set_docker_settings(config['init_config'], config['instances'][0])
+
+        self.run_check(config, force_reload=True)
+        self.assertServiceCheck('docker.container_health', at_least=2)
+
+
     def test_container_size(self):
         expected_metrics = [
             ('docker.containers.running', ['docker_image:nginx:latest', 'image_name:nginx', 'image_tag:latest']),