better handling of variing amount of fields in /proc/stat (#407)

Turns out we can't rely on all fields always being available, so we
need to be a little more defensive when parsing /proc/stat

fixes #406
closes #407

Author: beniwohli

CHANGELOG.md

@@ -9,6 +9,7 @@
  * Added some randomness to time between requests to APM Server (#426)
  * Fixed an issue with custom user models in Django using non-string usernames (#397, #398)
  * Fixed an issue with sending kubernetes metadata to the API (#401, #402)
+ * Fixed an issue with parsing /proc/stat in RHEL/centos 6 (#406, #407)
  * Added copyright header to all files, and a CI check (#429)
 
 ## v4.1.0 

elasticapm/metrics/sets/cpu_linux.py

@@ -39,6 +39,8 @@
 MEM_STATS = "/proc/meminfo"
 PROC_STATS = "/proc/self/stat"
 
+CPU_FIELDS = ("user", "nice", "system", "idle", "iowait", "irq", "softirq", "steal", "guest", "guest_nice")
+
 whitespace_re = re.compile(r"\s+")
 
 
@@ -84,11 +86,22 @@ def read_system_stats(self):
         with open(self.sys_stats_file, "r") as pidfile:
             for line in pidfile:
                 if line.startswith("cpu "):
-                    user, nice, system, idle, iowait, irq, softirq, steal, guest, guest_nice = map(
-                        int, whitespace_re.split(line)[1:-1]
+                    fields = whitespace_re.split(line)[1:-1]
+                    num_fields = len(fields)
+                    # Not all fields are available on all platforms (e.g. RHEL 6 does not provide steal, guest, and
+                    # guest_nice. If a field is missing, we default to 0
+                    f = {field: int(fields[i]) if i < num_fields else 0 for i, field in enumerate(CPU_FIELDS)}
+                    stats["cpu_total"] = float(
+                        f["user"]
+                        + f["nice"]
+                        + f["system"]
+                        + f["idle"]
+                        + f["iowait"]
+                        + f["irq"]
+                        + f["softirq"]
+                        + f["steal"]
                     )
-                    stats["cpu_total"] = float(user + nice + system + idle + iowait + irq + softirq + steal)
-                    stats["cpu_usage"] = stats["cpu_total"] - (idle + iowait)
+                    stats["cpu_usage"] = stats["cpu_total"] - (f["idle"] + f["iowait"])
                     break
         with open(self.memory_stats_file, "r") as memfile:
             for line in memfile:

tests/metrics/cpu_linux_tests.py

@@ -39,15 +39,11 @@
 
 
 TEMPLATE_PROC_STAT_SELF = """32677 (python) R 5333 32677 5333 34822 32677 4194304 13815 176 2 0 {utime} {stime} 0 0 20 0 7 0 6010710 3686981632 11655 18446744073709551615 94580847771648 94580849947501 140732830512176 0 0 0 0 16781312 134217730 0 0 0 17 1 0 0 6 0 0 94580850578256 94580851016824 94580875862016 140732830518932 140732830518950 140732830518950 140732830523339 0"""
-TEMPLATE_PROC_STAT = """cpu  {user} 2037 278561 {idle} 15536 0 178811 0 0 0
+
+
+TEMPLATE_PROC_STAT_DEBIAN = """cpu  {user} 2037 278561 {idle} 15536 0 178811 0 0 0
 cpu0 189150 166 34982 1081369 2172 0 73586 0 0 0
 cpu1 190286 201 35110 637359 1790 0 41941 0 0 0
-cpu2 191041 229 34494 636805 1782 0 21446 0 0 0
-cpu3 192144 309 34143 635312 1808 0 16014 0 0 0
-cpu4 190204 348 33770 636288 1880 0 9717 0 0 0
-cpu5 188740 333 33741 638328 1899 0 6939 0 0 0
-cpu6 188475 245 34417 635169 2323 0 3987 0 0 0
-cpu7 184451 204 37902 638971 1879 0 5178 0 0 0
 intr 60591079 11 12496 0 0 0 0 0 0 1 27128 0 0 474144 0 0 0 0 0 0 0 0 0 0 
 ctxt 215687788
 btime 1544981001
@@ -56,6 +52,18 @@
 procs_blocked 1
 softirq 61270136 2508862 17040144 383 7315609 6627 56 23515 19184746 49637 15140557
 """
+
+
+TEMPLATE_PROC_STAT_RHEL = """cpu  {user} 2037 278561 {idle} 15536 0 178811
+cpu0 259246 7001 60190 34250993 137517 772 0
+intr 354133732 347209999 2272 0 4 4 0 0 3 1 1249247 0 0 80143 0 422626 5169433
+ctxt 12547729
+btime 1093631447
+processes 130523
+procs_running 1
+procs_blocked 0"""
+
+
 TEMPLATE_PROC_MEMINFO = """MemTotal:       16164884 kB
 MemFree:          359184 kB
 MemAvailable:    6514428 kB
@@ -67,13 +75,14 @@
 """
 
 
-def test_cpu_mem_from_proc(tmpdir):
+@pytest.mark.parametrize("proc_stat_template", [TEMPLATE_PROC_STAT_DEBIAN, TEMPLATE_PROC_STAT_RHEL])
+def test_cpu_mem_from_proc(proc_stat_template, tmpdir):
     proc_stat_self = os.path.join(tmpdir.strpath, "self-stat")
     proc_stat = os.path.join(tmpdir.strpath, "stat")
     proc_meminfo = os.path.join(tmpdir.strpath, "meminfo")
 
     for path, content in (
-        (proc_stat, TEMPLATE_PROC_STAT.format(user=0, idle=0)),
+        (proc_stat, proc_stat_template.format(user=0, idle=0)),
         (proc_stat_self, TEMPLATE_PROC_STAT_SELF.format(utime=0, stime=0)),
         (proc_meminfo, TEMPLATE_PROC_MEMINFO),
     ):
@@ -84,7 +93,7 @@ def test_cpu_mem_from_proc(tmpdir):
     )
 
     for path, content in (
-        (proc_stat, TEMPLATE_PROC_STAT.format(user=400000, idle=600000)),
+        (proc_stat, proc_stat_template.format(user=400000, idle=600000)),
         (proc_stat_self, TEMPLATE_PROC_STAT_SELF.format(utime=100000, stime=100000)),
         (proc_meminfo, TEMPLATE_PROC_MEMINFO),
     ):