merged with latest master

Author: skp-github

.gitignore

@@ -49,4 +49,5 @@ opencamera-extended-firebase-adminsdk-yv5yz-e33a8ce5c1.json
 
 #
 # Project specific
+venv
 PythonTools/uploads

CaptureSync.iml

@@ -0,0 +1,118 @@
+import argparse
+from pathlib import Path
+import subprocess
+import math
+
+from video import video_info
+
+from typing import List, Tuple
+
+# See https://ffmpeg.org/ffmpeg-filters.html#xstack-1 for documentation on the xstack filter
+# Example (assuming HD input videos):
+"""
+ffmpeg -y -i video1.mp4 \
+-i video2.mp4 \
+-i video3.mp4 \
+-filter_complex \
+"\
+[0:v]scale=w=960:h=540[v0]; \
+[1:v]scale=w=960:h=540[v1]; \
+[2:v]scale=w=960:h=540[v2]; \
+[v0][v1][v2]xstack=inputs=3:layout=0_0|960_0|0_540[v]\
+" \
+-map "[v]" \
+finalOutput.mp4
+"""
+
+
+def create_video_grid_collage(video_files: List[str], output_file: str, grid_size: Tuple[int, int]) -> None:
+    # Determine the dimensions of each grid cell
+    grid_width, grid_height = grid_size
+    input_width, input_height, _ = video_info(video_files[0])
+    cell_width = input_width // grid_width
+    cell_height = input_height // grid_height
+
+    # Compose the ffmpeg command to create the collage
+    cmd = [
+        "ffmpeg",
+        "-y",  # Overwrite output file if it exists
+    ]
+
+    # Set the list of input videos
+    for i, video_file in enumerate(video_files):
+        cmd.extend(["-i", video_file])
+
+    #
+    # Generate the ffmpeg filtergraph
+    filtergraph = ""
+
+    # First, created the scaled video instances
+    for i, video_file in enumerate(video_files):
+        filtergraph += f"[{i}:v]scale=w={cell_width}:h={cell_height}[v{i}];"
+
+    # Now compose the layout for the xstack filter
+    xscale_positions = []
+    for i, video_file in enumerate(video_files):
+        col = i % grid_width
+        row = i // grid_width
+        xscale_positions.append(f"{cell_width*col}_{cell_height*row}")
+
+    filtergraph += "".join([f"[v{i}]" for i in range(len(video_files))])
+    filtergraph += f"xstack=inputs={len(xscale_positions)}:layout="
+    filtergraph += "|".join(xscale_positions)  # Compose the xstack layout (e.g.: "0_0|960_0|0_540")
+    filtergraph += "[v]"
+
+    # Append the complex filter and the remaining parameters
+    cmd.extend([
+        "-filter_complex", filtergraph,
+        "-map", "[v]",
+        "-c:v", "libx264",
+        "-crf", "18",
+        "-preset", "fast",
+        "-pix_fmt", "yuv420p",
+        output_file
+    ])
+    print("CMD:", cmd)
+    subprocess.run(cmd, capture_output=False)
+
+
+#
+# MAIN
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(
+        description="Fixes the videos produced by the RecSync recording sessions."
+                    "Converts the input recorder videos into videos with the same number of frames,"
+                    "with missing/dropped frames inserted as (black) artificial data."
+    )
+    parser.add_argument(
+        "--infolder", "-i", type=str, help="The folder containing the fixed videos.",
+        required=True
+    )
+    parser.add_argument(
+        "--outvideo", "-o", type=str, help="The filename to save the generated video.",
+        required=True
+    )
+
+    args = parser.parse_args()
+
+    input_dir = Path(args.infolder)
+    out_filename = args.outvideo
+
+    input_video_paths = input_dir.glob("*.mp4")
+    video_list = [str(p) for p in input_video_paths]
+    # print("Input videos: ", video_list)
+
+    # Compute the most squared grid to contain the given number of input videos (drops a row if needed)
+    n_videos = len(video_list)
+    n_cols = math.ceil(math.sqrt(n_videos))
+    n_rows = math.ceil(n_videos / n_cols)
+    # print(f"Composing grid of (cols X rows): {n_cols}X{n_rows}")
+
+    # Number of rows will be the same as number of columns, or one less
+    assert n_rows * n_cols >= n_videos
+    assert n_cols - 1 <= n_rows <= n_cols
+
+    grid_size = (n_cols, n_rows)
+
+    create_video_grid_collage(video_list, out_filename, grid_size)

PythonTools/CollateVideos.py

@@ -20,7 +20,7 @@
 #
 #
 #
-def main(input_dir: Path, output_dir: Path, threshold_ns: int):
+def post_process(input_dir: Path, output_dir: Path, threshold_ns: int = DEFAULT_THRESHOLD_NANOS):
 
     print(f"Scanning dir {str(input_dir)}...")
     clientIDs, df_list, mp4_list = scan_session_dir(input_dir)
@@ -96,7 +96,7 @@ def main(input_dir: Path, output_dir: Path, threshold_ns: int):
 
     parser = argparse.ArgumentParser(
         description="Fixes the videos produced by the RecSync recording sessions."
-                    "Output videos will have the same number of frames,"
+                    "Converts the input recorder videos into videos with the same number of frames,"
                     "with missing/dropped frames inserted as (black) artificial data."
     )
     parser.add_argument(
@@ -107,6 +107,10 @@ def main(input_dir: Path, output_dir: Path, threshold_ns: int):
         "--outfolder", "-o", type=str, help="The folder where the repaired and aligned frames will be stored.",
         required=True
     )
+    parser.add_argument(
+        "--create-outfolder", "-co", action="store_true", help="If the output folder doesn't exist, tries to create it.",
+        required=True
+    )
     parser.add_argument(
         "--threshold", "-t", type=int, help="The allowed difference in ms between corresponding frames on different videos."
                                             " Increase this is post processing fails with trimmed tables of different sizes."
@@ -119,13 +123,17 @@ def main(input_dir: Path, output_dir: Path, threshold_ns: int):
 
     infolder = Path(args.infolder)
     outfolder = Path(args.outfolder)
+    create_outfolder = args.create_outfolder
     threshold_millis = args.threshold
     threshold_nanos = threshold_millis * 1000 * 1000
 
     if not infolder.exists():
         raise Exception(f"Input folder '{infolder}' doesn't exist.")
 
     if not outfolder.exists():
-        raise Exception(f"Output folder '{outfolder}' doesn't exist.")
+        if create_outfolder:
+            outfolder.mkdir(parents=True, exist_ok=True)
+        else:
+            raise Exception(f"Output folder '{outfolder}' doesn't exist.")
 
-    main(infolder, outfolder, threshold_nanos)
+    post_process(input_dir=infolder, output_dir=outfolder, threshold_ns=threshold_nanos)

PythonTools/Makefile renamed to PythonTools/PostProcessDirectory.mk

@@ -3,8 +3,8 @@
 This utility "repairs" the videos generated by the RecSyncNG app:
 * Videos are unpacked
 * Missing frames are injected as black frames
-* Frame sequences are "synchronized" to have the same start time (with tolerance) and number of frames
-* Videos are re-packed, and they will all hve the same starting frame and the same number of frames
+* Frame sequences are "synchronized" to have the same number of frames and the same start time (with tolerance) 
+* Videos are re-packed, and they will all have the same starting frame and the same number of frames
   * A frame counter is added
 
 ## Installing

PythonTools/PostProcessVideos.py

@@ -164,11 +164,18 @@ def askStatus(self):
     def parseStatusInfo(self, status: str) -> None:
         """Extract the leader ID from the status text."""
         # E.g.: "Leader b308: 0 clients."
-        pattern = "^Leader ([0-9a-f][0-9a-f][0-9a-f][0-9a-f]): .*"
+        pattern = "^Leader ([0-9a-f][0-9a-f][0-9a-f][0-9a-f]).*"
+        # Alternative pattern, when there are no clients.
+        pattern2 = "^Leader : ([0-9a-f][0-9a-f][0-9a-f][0-9a-f])"
         res = re.match(pattern=pattern, string=status)
+        res2 = re.match(pattern=pattern2, string=status)
+
         if res is not None:
             self.leaderID = res.group(1)
             print(f"Leader ID is {self.leaderID}")
+        elif res2 is not None:
+            self.leaderID = res2.group(1)
+            print(f"Leader (alone) ID is {self.leaderID}")
 
     def deleteRemoteContent(self):
         msgBox = QMessageBox()

PythonTools/README.md

@@ -0,0 +1,58 @@
+![Logo](https://imgur.com/YtJA0E2.png)
+
+### Usage:
+
+
+#### Leader smartphone setup
+
+1.  Start a Wi-Fi hotspot.
+2.  The app should display connected clients and buttons for recording control
+
+#### Client smartphones setup
+
+1.  Enable WiFi and connect to the Wi-Fi hotspot.
+
+#### Recording video
+
+1.  [Optional step] Press the ```calculate period``` button. The app will analyze frame stream and use the calculated frame period in further synchronization steps.
+2.  Adjust exposure and ISO to your needs.
+3.  Press the ```phase align``` button.
+4.  Press the ```record video``` button to start synchronized video recording.
+5.  Get videos from RecSync folder in smartphone root directory.
+
+#### Extraction and matching of the frames
+
+```
+Requirements:
+
+- Python
+- ffmpeg
+```
+
+1. Navigate to ```utils``` directory in the repository.
+2. Run ```./match.sh <VIDEO_1> <VIDEO_2>```.
+3. Frames will be extracted to directories ```output/1``` and ```output/2``` with timestamps in filenames, output directory will also contain ```match.csv``` file in the following format:
+    ```
+    timestamp_1(ns) timestamp_2(ns)
+    ```
+
+### Our contribution:
+
+- Integrated **synchronized video recording**
+- Scripts for extraction, alignment and processing of video frames
+- Experiment with flash blinking to evaluate video frames synchronization accuracy
+- Panoramic video demo with automated Hugin stitching
+
+### Panoramic video stitching demo
+
+### [Link to youtube demo video](https://youtu.be/W6iANtCuQ-o)
+
+- We provide scripts to **stitch 2 syncronized smatphone videos** with Hujin panorama CLI tools
+- Usage:
+    - Run ```./make_demo.sh {VIDEO_LEFT} {VIDEO_RIGHT}```
+
+### This work is based on "Wireless Software Synchronization of Multiple Distributed Cameras"
+
+Reference code for the paper
+[Wireless Software Synchronization of Multiple Distributed Cameras](https://arxiv.org/abs/1812.09366).
+_Sameer Ansari, Neal Wadhwa, Rahul Garg, Jiawen Chen_, ICCP 2019.