Clean ups and more tests. Reparation fails: leave holes. Trimming fails: doesn't work on repaired frames with headers.

Author: fnunnari

PostProcessing/PostProcessVideos.py

@@ -5,7 +5,10 @@
 import pandas as pd
 import re
 
-from dataframes import compute_time_range, trim_into_interval, repair_dropped_frames
+from dataframes import compute_time_range, trim_into_interval
+from dataframes import repair_dropped_frames, compute_time_step
+
+from video import extract_frames
 
 
 THRESHOLD_NS = 10 * 1000 * 1000
@@ -56,6 +59,7 @@ def scan_session_dir(input_dir: Path) -> Tuple[List[str], List[pd.DataFrame], Li
 
     return clientIDs, df_list, mp4_list
 
+
 #
 #
 #
@@ -76,10 +80,11 @@ def main(input_dir: Path, output_dir: Path):
         print(f"For client ID {cID}: {len(df)} frames for file {mp4}")
 
     #
-    # Repair CSVs (TODO - Mina)
+    # Repair CSVs
     repaired_df_list: List[pd.DataFrame] = []
     for cID, df in zip(clientIDs, df_list):
-        repaired_df = repair_dropped_frames(df)
+        time_step = compute_time_step(df)
+        repaired_df = repair_dropped_frames(df=df, time_step=time_step)
         repaired_df_list.append(repaired_df)
 
     assert len(clientIDs) == len(df_list) == len(mp4_list) == len(repaired_df_list)
@@ -90,7 +95,7 @@ def main(input_dir: Path, output_dir: Path):
     min_common, max_common = compute_time_range(repaired_df_list)
 
     #
-    # Trim CSVs (TODO)
+    # Trim CSVs
     # Trim the data frames to the time range
     trimmed_dataframes = trim_into_interval(repaired_df_list, min_common, max_common, THRESHOLD_NS)
 
@@ -109,7 +114,7 @@ def main(input_dir: Path, output_dir: Path):
     #
     # Extract the frames from the original videos
     # and rename the file names to the timestamps (DONE)
-    # extract(input_dir, output_dir)
+    # extract_frames(input_dir, output_dir)
 
     #
     # Reconstruct videos (TODO)

PostProcessing/dataframes.py

@@ -30,7 +30,7 @@ def repair_dropped_frames(df: pd.DataFrame, time_step: float) -> pd.DataFrame:
 
     # Forces the type of the timestamps to int64
     df[first_col_name] = pd.to_datetime(df[first_col_name]).astype(np.int64)
-    # Retrieves the timestamps into a Serie
+    # Retrieves the timestamps into a Series
     timestamps = df[first_col_name]
     # Will accumulate the repaired rows
     repaired_rows = []
@@ -114,11 +114,12 @@ def trim_into_interval(dfs, min_common, max_common, threshold):
     for df in dfs:
         start: pd.DataFrame = df[df.iloc[:, 0].between(min_common-threshold, min_common+threshold, inclusive='both')]
         end: pd.DataFrame = df[df.iloc[:, 0].between(max_common-threshold, max_common+threshold, inclusive='both')]
-        if not start.empty and not end.empty :
+        if not start.empty and not end.empty:
             df_start = start.stack().iloc[0]
             df_end = end.stack().iloc[-1]
             trimmed_df = df[df.iloc[:, 0].between(df_start, df_end, inclusive='both')]
             trimmed_dataframes.append(trimmed_df)
         else:
-            print("No values found within the specified range.")
+            raise Exception("No values found within the specified range.")
+
     return trimmed_dataframes

PostProcessing/test_functions.py

@@ -84,6 +84,12 @@ def test_df_reparation(client_data):
     assert df[first_col_name].iloc[-1] == repaired_df[repaired_first_col_name].iloc[-1],\
         "The last element changed after reparation"
 
+    tstamps = repaired_df["timestamp"]
+    diffs = tstamps.diff().dropna()
+    assert (diffs <= time_step).all(), "some timestamps difference is longer than expected"
+
+    pass
+
 
 def test_df_trimming(session_data):
     _, dataframes, _ = session_data