Merge branch 'remotecontrolling' of https://github.com/DFKI-SignLanguage/RecSync-android into remotecontrolling

Author: skp-github

PostProcessing/PostProcessVideos.py

@@ -11,9 +11,11 @@
 
 from video import extract_frames
 from video import rebuild_video
+from video import extract_video_info
 
 
-THRESHOLD_NS = 10 * 1000 * 1000  # millis * micros * nanos
+DEFAULT_THRESHOLD_MILLIS = 10
+DEFAULT_THRESHOLD_NANOS = DEFAULT_THRESHOLD_MILLIS * 1000 * 1000  # millis * micros * nanos
 
 
 def scan_session_dir(input_dir: Path) -> Tuple[List[str], List[pd.DataFrame], List[str]]:
@@ -65,14 +67,13 @@ def scan_session_dir(input_dir: Path) -> Tuple[List[str], List[pd.DataFrame], Li
 #
 #
 #
-def main(input_dir: Path, output_dir: Path):
+def main(input_dir: Path, output_dir: Path, threshold_ns: int):
 
     print(f"Scanning dir {str(input_dir)}...")
     clientIDs, df_list, mp4_list = scan_session_dir(input_dir)
 
     n_clients = len(clientIDs)
 
-
     #
     # Print collected info
     for i in range(n_clients):
@@ -96,18 +97,21 @@ def main(input_dir: Path, output_dir: Path):
     # Find time ranges
     min_common, max_common = compute_time_range(repaired_df_list)
     # Trim the data frames to the time range
-    trimmed_dataframes = trim_repaired_into_interval(repaired_df_list, min_common, max_common, THRESHOLD_NS)
+    trimmed_dataframes = trim_repaired_into_interval(repaired_df_list, min_common, max_common, threshold_ns)
 
     assert len(clientIDs) == len(trimmed_dataframes), f"Expected {len(clientIDs)} trimmed dataframes. Found f{len(trimmed_dataframes)}"
 
     # Check that all the resulting dataframes have the same number of rows
+    print("Checking if all clients we have the same number of frames in the repaired amd trimmed tables...")
     client0ID = clientIDs[0]
     client0size = len(trimmed_dataframes[0])
-    print(f"For client {client0ID}: {client0size} frames")
+    print(f"Client {client0ID} has {client0size} frames.")
     for cID, df in zip(clientIDs[1:], trimmed_dataframes[1:]):
         dfsize = len(df)
         if client0size != dfsize:
-            raise Exception(f"For client {cID}: expecting {client0size} frames, found {dfsize}")
+            raise Exception(f"For client {cID}: expecting {client0size} frames. Found {dfsize}."
+                            f" This might be due to an excessive phase offset during recording."
+                            f" Try to increase the threshold.")
 
     print("Good. All trimmed dataframes have the same number of entries.")
 
@@ -125,8 +129,10 @@ def main(input_dir: Path, output_dir: Path):
             extract_frames(video_file=video_file, timestamps_df=orig_df, output_dir=tmp_dir)
 
             # Reconstruct videos
+            vinfo = extract_video_info(video_path=video_file)
+            input_fps = vinfo.fps
             video_out_filepath = output_dir / (cID + ".mp4")
-            rebuild_video(dir=Path(tmp_dir), frames=trimmed_df, outfile=video_out_filepath)
+            rebuild_video(dir=Path(tmp_dir), frames=trimmed_df, fps=input_fps, outfile=video_out_filepath)
             # And save also the CSV
             csv_out_filepath = video_out_filepath.with_suffix(".csv")
             trimmed_df.to_csv(path_or_buf=csv_out_filepath, header=True, index=False)
@@ -149,16 +155,25 @@ def main(input_dir: Path, output_dir: Path):
         "--outfolder", "-o", type=str, help="The folder where the repaired and aligned frames will be stored.",
         required=True
     )
+    parser.add_argument(
+        "--threshold", "-t", type=int, help="The allowed difference in ms between corresponding frames on different videos."
+                                            " Increase this is post processing fails with trimmed tables of different sizes."
+                                            f" Default is now {DEFAULT_THRESHOLD_MILLIS} ms.",
+        required=False,
+        default=DEFAULT_THRESHOLD_MILLIS
+    )
 
     args = parser.parse_args()
 
     infolder = Path(args.infolder)
     outfolder = Path(args.outfolder)
+    threshold_millis = args.threshold
+    threshold_nanos = threshold_millis * 1000 * 1000
 
     if not infolder.exists():
         raise Exception(f"Input folder '{infolder}' doesn't exist.")
 
     if not outfolder.exists():
         raise Exception(f"Output folder '{outfolder}' doesn't exist.")
 
-    main(infolder, outfolder)
+    main(infolder, outfolder, threshold_nanos)

PostProcessing/dataframes.py

@@ -120,4 +120,6 @@ def trim_repaired_into_interval(dfs, min_common, max_common, threshold) -> List[
         trimmed_df = df[selection_mask]
         trimmed_dataframes.append(trimmed_df)
 
+        assert len(trimmed_dataframes) <= len(df)
+
     return trimmed_dataframes

PostProcessing/test_functions.py

@@ -13,7 +13,7 @@
 
 from video import video_info
 
-from PostProcessVideos import THRESHOLD_NS
+from PostProcessVideos import DEFAULT_THRESHOLD_NANOS
 
 
 RECSYNCH_SESSION_DIR_VAR = "RECSYNCH_SESSION_DIR"
@@ -94,7 +94,7 @@ def test_df_reparation(client_data):
 
     tstamps = repaired_df["timestamp"]
     diffs = tstamps.diff().dropna()
-    assert (diffs <= (time_step + THRESHOLD_NS)).all(), "some timestamps difference is longer than expected"
+    assert (diffs <= (time_step + DEFAULT_THRESHOLD_NANOS)).all(), "some timestamps difference is longer than expected"
 
 
 def test_df_trimming(session_data):

PostProcessing/video.py

@@ -1,6 +1,7 @@
 import cv2
 import os
 from pathlib import Path
+from collections import namedtuple
 
 import pandas as pd
 import numpy as np
@@ -35,7 +36,43 @@ def video_info(video_path: str) -> Tuple[int, int, int]:
     return video_w, video_h, n_frames
 
 
-def extract_frames(video_file: str, timestamps_df: pd.DataFrame, output_dir: str):
+VideoInfo = namedtuple("VideoInfo", ["width", "height", "n_frames", "fps", "codec"])
+
+
+def extract_video_info(video_path: str) -> VideoInfo:
+    """
+    Uses the ffmpeg.probe function to retrieve information about a video file.
+
+    :param video_path: Path to a valid video file
+    :return: An instance of a VideoTuple named tuple with self-explaining information.
+    """
+
+    #
+    # Fetch video info
+    info = ffmpeg.probe(video_path)
+    # Get the list of all video streams
+    video_streams = [stream for stream in info['streams'] if stream['codec_type'] == 'video']
+    if len(video_streams) == 0:
+        raise BaseException("No video streams found in file '{}'".format(video_path))
+
+    # retrieve the first stream of type 'video'
+    info_video = video_streams[0]
+
+    framerate_ratio_str = info_video['r_frame_rate']
+    fps = eval(framerate_ratio_str)
+
+    out = VideoInfo(
+        width=info_video['width'],
+        height=info_video['height'],
+        n_frames=int(info_video['nb_frames']),
+        fps=fps,
+        codec=info_video['codec_name']
+    )
+
+    return out
+
+
+def extract_frames(video_file: str, timestamps_df: pd.DataFrame, output_dir: str) -> None:
 
     # Open the video file
     cap = cv2.VideoCapture(video_file)
@@ -60,7 +97,7 @@ def extract_frames(video_file: str, timestamps_df: pd.DataFrame, output_dir: str
     cap.release()
 
 
-def extract_frames_ffmpeg(video_file: str, timestamps_df: pd.DataFrame, output_dir: str):
+def extract_frames_ffmpeg(video_file: str, timestamps_df: pd.DataFrame, output_dir: str) -> None:
     video_w, video_h, _ = video_info(video_file)
 
     ffmpeg_read_process = (
@@ -98,7 +135,7 @@ def extract_frames_ffmpeg(video_file: str, timestamps_df: pd.DataFrame, output_d
         ffmpeg_read_process = None
 
 
-def rebuild_video(dir: Path, frames: pd.DataFrame, outfile: Path) -> None:
+def rebuild_video(dir: Path, frames: pd.DataFrame, fps: float, outfile: Path) -> None:
 
     # We don't know the target video size, yet.
     frame_width = None
@@ -131,6 +168,7 @@ def rebuild_video(dir: Path, frames: pd.DataFrame, outfile: Path) -> None:
                 ffmpeg_video_out_process = (
                                     ffmpeg
                                         .input('pipe:', format='rawvideo', pix_fmt='rgb24', s='{}x{}'.format(frame_width, frame_height))
+                                        .filter('fps', fps=fps)
                                         # -vf "drawtext=fontfile=Arial.ttf: fontsize=48: text=%{n}: x=(w-tw)/2: y=h-(2*lh): fontcolor=white: box=1: boxcolor=0x00000099"
                                         .drawtext(text="%{n}", escape_text=False,
                                                   #x=50, y=50,

remote_control/remote_control.py

@@ -222,9 +222,9 @@ def retranslateUi(self, MainWindow):
         self.stop_btn.setText(_translate("MainWindow", "Stop"))
         self.status_btn.setText(_translate("MainWindow", "Status"))
         self.status_clear_btn.setText(_translate("MainWindow", "X"))
-        self.status_clear_btn.setStyleSheet('QPushButton {;color: red;}')
+        self.status_clear_btn.setStyleSheet('QPushButton {;color: #cc2222;}')
         self.delete_btn.setText(_translate("MainWindow", "Empty Device"))
-        self.delete_btn.setStyleSheet('QPushButton {;background-color: red;}')
+        self.delete_btn.setStyleSheet('QPushButton {;background-color: #cc2222;}')
         self.api_input.setPlaceholderText(_translate("MainWindow", "Please enter the api endpoint where you want the files to be uploaded."))
         self.download_prefix_text.setPlaceholderText(_translate("MainWindow", " Enter Session Prefix"))
         self.download_btn.setText(_translate("MainWindow", "Download"))
@@ -240,4 +240,5 @@ def retranslateUi(self, MainWindow):
     rc.setupUi(MainWindow)
 
     MainWindow.show()
+
     sys.exit(app.exec_())