Diagnose detects discarded frames

Author: Bercon

python/cli/diagnose/diagnose.py

@@ -54,7 +54,8 @@ def generateReport(args):
             "velocity": [], # ENU
             "altitude": []  # WGS-84
         }},
-        'cameras': {}
+        'cameras': {},
+        'discardedFrames': []
     }
 
     def addMeasurement(type, t, v):
@@ -88,6 +89,7 @@ def addMeasurement(type, t, v):
             frames = measurement.get("frames")
             metrics = measurement.get("systemMetrics")
             vioOutput = measurement if "status" in measurement else None
+            droppedFrame = measurement.get("droppedFrame")
             if frames is None and 'frame' in measurement:
                 frames = [measurement['frame']]
                 frames[0]['cameraInd'] = 0
@@ -99,7 +101,8 @@ def addMeasurement(type, t, v):
                 and metrics is None
                 and barometer is None
                 and gnss is None
-                and vioOutput is None): continue
+                and vioOutput is None
+                and droppedFrame is None): continue
 
             if startTime is None:
                 startTime = time
@@ -184,6 +187,9 @@ def addMeasurement(type, t, v):
                     vio["global"]["velocity"].append([globalPose["velocity"][c] for c in "xyz"])
                     vio["global"]["altitude"].append(wgs84["altitude"])
 
+            elif droppedFrame:
+                data['discardedFrames'].append(time)
+
         if nSkipped > 0: print(f'Skipped {nSkipped} lines')
 
     diagnoseCamera(data, output)

python/cli/diagnose/html.py

@@ -192,6 +192,9 @@ def generateHtml(output, outputHtml):
         for camera in output["cameras"]:
             s += generateSensor(camera, 'Camera #{}'.format(camera["ind"]))
 
+    if output.get("discardedFrames"):
+        s += generateSensor(output.get("discardedFrames"), 'Cameras')
+
     for sensor in SENSOR_NAMES:
         if sensor not in output: continue
         name = sensor.capitalize() if sensor.islower() else sensor

python/cli/diagnose/sensors.py

@@ -71,6 +71,29 @@ def plotFrame(
 
     return base64(fig)
 
+
+def plotDiscardedFrames(
+        start,
+        end,
+        data):
+    import matplotlib.pyplot as plt
+    from matplotlib.ticker import ScalarFormatter
+
+    fig, ax = plt.subplots(figsize=(8, 2))
+
+    ax.set_title("Discarded frames")
+    ax.vlines(data, ymin=0, ymax=1, colors='red', lw=1)
+    ax.set_xlim(start, end)
+    ax.set_ylim(0, 1)
+    ax.set_xlabel("Time (s)")
+    ax.get_yaxis().set_visible(False)
+    ax.xaxis.set_major_formatter(ScalarFormatter())
+    ax.ticklabel_format(style='plain',axis='x',useOffset=False)
+    fig.tight_layout()
+
+    return base64(fig)
+
+
 class DiagnosisLevel(Enum):
     OK = 0
     WARNING = 1
@@ -210,6 +233,18 @@ def toPercent(value):
                     f"Found {invalidTimestamps} ({toPercent(invalidTimestamps)}) "
                     "timestamps that don't overlap with IMU")
 
+
+    def analyzeDiscardedFrames(self, start_time, end_time, timestamps):
+        if len(timestamps) == 0: return
+        self.images.append(plotDiscardedFrames(start_time, end_time, timestamps))
+        self.__addIssue(DiagnosisLevel.WARNING,
+            f"Found {len(timestamps)} discarded frames. Typically this happens when disk I/O "
+            f"is too slow and prevents recording frames leading them to be discarded. You can "
+            f"alleviate this issue with larger in-memory buffer when recording using FFMpeg. "
+            f"In <i>vio_config.yaml</i>: <i>ffmpegWriteBufferCapacityMegabytes: 50</i>"
+        )
+
+
     def analyzeArrivalTimes(
             self,
             timestamps,
@@ -652,13 +687,18 @@ def diagnoseCamera(data, output):
     sensor = data["cameras"]
     output["cameras"] = []
 
+    start_time = None
+    end_time = None
     for ind in sensor.keys():
         camera = sensor[ind]
         timestamps = np.array(camera["t"])
         deltaTimes = np.array(camera["td"])
 
         if len(timestamps) == 0: continue
 
+        if start_time == None or start_time > timestamps[0]: start_time = timestamps[0]
+        if end_time == None or end_time < timestamps[-1]: end_time = timestamps[-1]
+
         status = Status()
         status.analyzeTimestamps(
             timestamps,
@@ -700,6 +740,17 @@ def diagnoseCamera(data, output):
         # Camera is required
         output["passed"] = False
 
+    if len(data["discardedFrames"]) > 0:
+        status = Status()
+        status.analyzeDiscardedFrames(start_time, end_time, data["discardedFrames"])
+        output["discardedFrames"] = {
+            "diagnosis": status.diagnosis.toString(),
+            "issues": status.serializeIssues(),
+            "count": len(timestamps),
+            "images": status.images
+        }
+
+
 def diagnoseAccelerometer(data, output):
     ACC_MIN_FREQUENCY_HZ = 50.0
     ACC_MAX_FREQUENCY_HZ = 1e4