Add cgroup support (#831) (#846)

* Add cgroup support (#831)

* Fix search to cgroup directory, looking for where it is mounted

* Fix None

* Change metrics names

* Round fix

* adapted code style to pep8, added some comments and a couple more tests

* use tighter exception handling, and removed a couple of unnecessary `return None`

Co-authored-by: Benjamin Wohlwend <[email protected]>
Co-authored-by: Colton Myers <[email protected]>

Author: 3 people

elasticapm/metrics/sets/cpu_linux.py

@@ -28,6 +28,7 @@
 #  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 #  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import logging
 import os
 import re
 import resource
@@ -38,19 +39,42 @@
 SYS_STATS = "/proc/stat"
 MEM_STATS = "/proc/meminfo"
 PROC_STATS = "/proc/self/stat"
+CGROUP1_MEMORY_LIMIT = "memory.limit_in_bytes"
+CGROUP1_MEMORY_USAGE = "memory.usage_in_bytes"
+CGROUP1_MEMORY_STAT = "memory.stat"
+CGROUP2_MEMORY_LIMIT = "memory.max"
+CGROUP2_MEMORY_USAGE = "memory.current"
+CGROUP2_MEMORY_STAT = "memory.stat"
+UNLIMITED = 0x7FFFFFFFFFFFF000
+PROC_SELF_CGROUP = "/proc/self/cgroup"
+PROC_SELF_MOUNTINFO = "/proc/self/mountinfo"
+SYS_FS_CGROUP = "/sys/fs/cgroup"
 
 CPU_FIELDS = ("user", "nice", "system", "idle", "iowait", "irq", "softirq", "steal", "guest", "guest_nice")
 MEM_FIELDS = ("MemTotal", "MemAvailable", "MemFree", "Buffers", "Cached")
 
 whitespace_re = re.compile(r"\s+")
 
+MEMORY_CGROUP = re.compile(r"^\d+:memory:.*")
+CGROUP_V1_MOUNT_POINT = re.compile(r"^\d+? \d+? .+? .+? (.*?) .*cgroup.*memory.*")
+CGROUP_V2_MOUNT_POINT = re.compile(r"^\d+? \d+? .+? .+? (.*?) .*cgroup2.*cgroup.*")
 
 if not os.path.exists(SYS_STATS):
     raise ImportError("This metric set is only available on Linux")
 
+logger = logging.getLogger("elasticapm.metrics.cpu_linux")
+
 
 class CPUMetricSet(MetricsSet):
-    def __init__(self, registry, sys_stats_file=SYS_STATS, process_stats_file=PROC_STATS, memory_stats_file=MEM_STATS):
+    def __init__(
+        self,
+        registry,
+        sys_stats_file=SYS_STATS,
+        process_stats_file=PROC_STATS,
+        memory_stats_file=MEM_STATS,
+        proc_self_cgroup=PROC_SELF_CGROUP,
+        mount_info=PROC_SELF_MOUNTINFO,
+    ):
         self.page_size = resource.getpagesize()
         self.previous = {}
         self._read_data_lock = threading.Lock()
@@ -59,10 +83,91 @@ def __init__(self, registry, sys_stats_file=SYS_STATS, process_stats_file=PROC_S
         self.memory_stats_file = memory_stats_file
         self._sys_clock_ticks = os.sysconf("SC_CLK_TCK")
         with self._read_data_lock:
+            try:
+                self.cgroup_files = self.get_cgroup_file_paths(proc_self_cgroup, mount_info)
+            except Exception:
+                logger.debug("Reading/Parsing of cgroup memory files failed, skipping cgroup metrics", exc_info=True)
             self.previous.update(self.read_process_stats())
             self.previous.update(self.read_system_stats())
         super(CPUMetricSet, self).__init__(registry)
 
+    def get_cgroup_file_paths(self, proc_self_cgroup, mount_info):
+        """
+        Try and find the paths for CGROUP memory limit files, first trying to find the root path
+        in /proc/self/mountinfo, then falling back to the default location /sys/fs/cgroup
+        :param proc_self_cgroup: path to "self" cgroup file, usually /proc/self/cgroup
+        :param mount_info: path to "mountinfo" file, usually proc/self/mountinfo
+        :return: a 3-tuple of memory info files, or None
+        """
+        line_cgroup = None
+        try:
+            with open(proc_self_cgroup, "r") as proc_self_cgroup_file:
+                for line in proc_self_cgroup_file:
+                    if line_cgroup is None and line.startswith("0:"):
+                        line_cgroup = line
+                    if MEMORY_CGROUP.match(line):
+                        line_cgroup = line
+                        break
+        except IOError:
+            logger.debug("Cannot read %s, skipping cgroup metrics", proc_self_cgroup, exc_info=True)
+            return
+        if line_cgroup is None:
+            return
+        try:
+            with open(mount_info, "r") as mount_info_file:
+                for line in mount_info_file:
+                    # cgroup v2
+                    matcher = CGROUP_V2_MOUNT_POINT.match(line)
+                    if matcher is not None:
+                        files = self._get_cgroup_v2_file_paths(line_cgroup, matcher.group(1))
+                        if files:
+                            return files
+                    # cgroup v1
+                    matcher = CGROUP_V1_MOUNT_POINT.match(line)
+                    if matcher is not None:
+                        files = self._get_cgroup_v1_file_paths(matcher.group(1))
+                        if files:
+                            return files
+        except IOError:
+            logger.debug("Cannot read %s, skipping cgroup metrics", mount_info, exc_info=True)
+            return
+        # discovery of cgroup path failed, try with default path
+        files = self._get_cgroup_v2_file_paths(line_cgroup, SYS_FS_CGROUP)
+        if files:
+            return files
+        files = self._get_cgroup_v1_file_paths(os.path.join(SYS_FS_CGROUP, "memory"))
+        if files:
+            return files
+        logger.debug("Location of cgroup files failed, skipping cgroup metrics")
+
+    def _get_cgroup_v2_file_paths(self, line_cgroup, mount_discovered):
+        line_split = line_cgroup.strip().split(":")
+        slice_path = line_split[-1][1:]
+        try:
+            with open(os.path.join(mount_discovered, slice_path, CGROUP2_MEMORY_LIMIT), "r") as memfile:
+                line_mem = memfile.readline().strip()
+                if line_mem != "max":
+                    return (
+                        os.path.join(mount_discovered, slice_path, CGROUP2_MEMORY_LIMIT),
+                        os.path.join(mount_discovered, slice_path, CGROUP2_MEMORY_USAGE),
+                        os.path.join(mount_discovered, slice_path, CGROUP2_MEMORY_STAT),
+                    )
+        except IOError:
+            pass
+
+    def _get_cgroup_v1_file_paths(self, mount_discovered):
+        try:
+            with open(os.path.join(mount_discovered, CGROUP1_MEMORY_LIMIT), "r") as memfile:
+                mem_max = int(memfile.readline().strip())
+                if mem_max < UNLIMITED:
+                    return (
+                        os.path.join(mount_discovered, CGROUP1_MEMORY_LIMIT),
+                        os.path.join(mount_discovered, CGROUP1_MEMORY_USAGE),
+                        os.path.join(mount_discovered, CGROUP1_MEMORY_STAT),
+                    )
+        except IOError:
+            pass
+
     def before_collect(self):
         new = self.read_process_stats()
         new.update(self.read_system_stats())
@@ -83,6 +188,13 @@ def before_collect(self):
             self.gauge("system.memory.actual.free").val = mem_free
             self.gauge("system.memory.total").val = new["MemTotal"]
 
+            if "cgroup_mem_total" in new:
+                self.gauge("system.process.cgroup.memory.mem.limit.bytes").val = new["cgroup_mem_total"]
+            if "cgroup_mem_used" in new:
+                self.gauge("system.process.cgroup.memory.mem.usage.bytes").val = new["cgroup_mem_used"]
+            if "cgroup_mem_inactive" in new:
+                self.gauge("system.process.cgroup.memory.stats.inactive_file.bytes").val = new["cgroup_mem_inactive"]
+
             try:
                 cpu_process_percent = delta["proc_total_time"] / delta["cpu_total"]
             except ZeroDivisionError:
@@ -115,6 +227,20 @@ def read_system_stats(self):
                     )
                     stats["cpu_usage"] = stats["cpu_total"] - (f["idle"] + f["iowait"])
                     break
+        if self.cgroup_files:
+            with open(self.cgroup_files[0], "r") as memfile:
+                stats["cgroup_mem_total"] = int(memfile.readline())
+            with open(self.cgroup_files[1], "r") as memfile:
+                usage = int(memfile.readline())
+                stats["cgroup_mem_used"] = usage
+            with open(self.cgroup_files[2], "r") as memfile:
+                sum = 0
+                for line in memfile:
+                    (metric_name, value) = line.split(" ")
+                    if metric_name == "inactive_file":
+                        sum = sum + int(value)
+                stats["cgroup_mem_used"] = stats["cgroup_mem_used"] - sum
+                stats["cgroup_mem_inactive"] = sum
         with open(self.memory_stats_file, "r") as memfile:
             for line in memfile:
                 metric_name = line.split(":")[0]