misc: update README.md for dynamic splats-related scripts

Author: JasonLSC

README.md

@@ -60,14 +60,25 @@ pip install git+https://github.com/fraunhoferhhi/PLAS.git
 We provide a script that enables more memory-efficient Gaussian splats while maintaining high visual quality, such as representing the Truck scene with only about 8MB of storage. The script includes 1) the static splats training with compression simulation, 2) the compression of trained static splats, and 3) the metric evaluation of uncompressed and compressed static splats.
 
 ```bash
+# Tanks and Temples dataset
 bash benchmarks/compression/final_exp/mcmc_tt_sim.sh
 ```
 
 **Dynamic Gaussian Splats Training and Compression**
-(Will be provided on 2/15/2025)
+First, please follow the dataset preprocessing instruction described in the [file]((mpeg_gsc_utils/multiview_video_preprocess/README.md)) for training data prepration.
+
+Next, run the script for dynamic gaussian splats training and compression.
+```bash
+cd examples
+bash benchmarks/dyngs/dyngs.sh
+```
 
 **Extract Per-Frame Static Gaussian from Dynamic Splats**
-(Will be provided on 2/15/2025)
+If you finsh the training of dynamic splats, then you can use the script to extract static gaussian splats stored at discrte timesteps in ".ply" file
+```bash
+cd examples
+bash benchmarks/dyngs/export_plys.sh
+```
 
 ## Contributors
 

examples/benchmarks/compression/final_exp/mcmc_tt_sim.sh

@@ -41,15 +41,15 @@ run_single_scene() {
     echo "Running $SCENE on GPU: $GPU_ID"
 
     # train without eval
-    CUDA_VISIBLE_DEVICES=$GPU_ID python simple_trainer.py mcmc --eval_steps -1 --disable_viewer --data_factor 1 \
-        --strategy.cap-max $CAP_MAX \
-        --data_dir $SCENE_DIR/$SCENE/ \
-        --result_dir $RESULT_DIR/$SCENE/ \
-        --compression_sim \
-        --entropy_model_opt \
-        --rd_lambda $RD_LAMBDA \
-        --shN_ada_mask_opt \
-        --compression png
+    # CUDA_VISIBLE_DEVICES=$GPU_ID python simple_trainer.py mcmc --eval_steps -1 --disable_viewer --data_factor 1 \
+    #     --strategy.cap-max $CAP_MAX \
+    #     --data_dir $SCENE_DIR/$SCENE/ \
+    #     --result_dir $RESULT_DIR/$SCENE/ \
+    #     --compression_sim \
+    #     --entropy_model_opt \
+    #     --rd_lambda $RD_LAMBDA \
+    #     --shN_ada_mask_opt \
+    #     --compression png
 
 
     # eval: use vgg for lpips to align with other benchmarks

examples/benchmarks/dyngs/dyngs.sh

@@ -0,0 +1,61 @@
+SCENE_DIR="data/GSC"
+SCENE_LIST="Bartender" # CBA Bartender
+
+declare -A TEST_VIEWS
+TEST_VIEWS=(
+    ["CBA"]="7 22"
+    ["Bartender"]="8 10 12"
+)
+
+RESULT_DIR="results/dyngs"
+
+NUM_FRAME=65
+
+run_single_scene() {
+    local GPU_ID=$1
+    local SCENE=$2
+    local TEST_VIEW_IDS=${TEST_VIEWS[$SCENE]}
+
+    echo "Running $SCENE"
+
+    CUDA_VISIBLE_DEVICES=$GPU_ID python simple_trainer_dyngs.py compression_sim \
+        --model_path $RESULT_DIR/$SCENE/ \
+        --data_dir $SCENE_DIR/$SCENE/colmap/colmap_50 \
+        --result_dir $RESULT_DIR/$SCENE/ \
+        --downscale_factor 1 \
+        --duration $NUM_FRAME \
+        --batch_size 2 \
+        --max_steps 60_000 \
+        --refine_start_iter 3_000 \
+        --refine_stop_iter 30_000 \
+        --refine_every 100 \
+        --reset_every 6_000 \
+        --pause_refine_after_reset 500 \
+        --strategy Modified_STG_Strategy \
+        --test_view_id $TEST_VIEW_IDS 
+    
+    CUDA_VISIBLE_DEVICES=$GPU_ID python simple_trainer_dyngs.py default \
+        --model_path $RESULT_DIR/$SCENE/ \
+        --data_dir $SCENE_DIR/$SCENE/colmap/colmap_50 \
+        --result_dir $RESULT_DIR/$SCENE/ \
+        --downscale_factor 1 \
+        --duration $NUM_FRAME \
+        --lpips_net vgg \
+        --compression stg \
+        --ckpt $RESULT_DIR/$SCENE/ckpts/ckpt_best_rank0.pt \
+        --test_view_id $TEST_VIEW_IDS 
+}
+
+GPU_LIST=(7)
+GPU_COUNT=${#GPU_LIST[@]}
+
+SCENE_IDX=-1
+
+for SCENE in $SCENE_LIST;
+do
+    SCENE_IDX=$((SCENE_IDX + 1))
+    {
+        run_single_scene ${GPU_LIST[$SCENE_IDX]} $SCENE
+    } #&
+
+done

examples/benchmarks/dyngs/export_plys.sh

@@ -0,0 +1,46 @@
+SCENE_DIR="data/GSC"
+SCENE_LIST="Bartender" # CBA Bartender
+
+declare -A TEST_VIEWS
+TEST_VIEWS=(
+    ["CBA"]="7 22"
+    ["Bartender"]="8 10 12"
+)
+
+RESULT_DIR="results/dyngs"
+
+NUM_FRAME=65
+
+run_single_scene() {
+    local GPU_ID=$1
+    local SCENE=$2
+    local TEST_VIEW_IDS=${TEST_VIEWS[$SCENE]}
+
+    echo "Running $SCENE"
+    
+    CUDA_VISIBLE_DEVICES=$GPU_ID python simple_trainer_dyngs.py default \
+        --model_path $RESULT_DIR/$SCENE/ \
+        --data_dir $SCENE_DIR/$SCENE/colmap/colmap_50 \
+        --result_dir $RESULT_DIR/$SCENE/ \
+        --downscale_factor 1 \
+        --duration $NUM_FRAME \
+        --lpips_net vgg \
+        --compression stg \
+        --ckpt $RESULT_DIR/$SCENE/ckpts/ckpt_best_rank0.pt \
+        --test_view_id $TEST_VIEW_IDS \
+        --enable_dyn_splats_export
+}
+
+GPU_LIST=(7)
+GPU_COUNT=${#GPU_LIST[@]}
+
+SCENE_IDX=-1
+
+for SCENE in $SCENE_LIST;
+do
+    SCENE_IDX=$((SCENE_IDX + 1))
+    {
+        run_single_scene ${GPU_LIST[$SCENE_IDX]} $SCENE
+    } #&
+
+done

examples/helper/STG/dataset_readers.py

@@ -135,7 +135,6 @@ def readColmapCameras(cam_extrinsics, cam_intrinsics, images_folder, near, far,
         uid = intr.id
         R = np.transpose(qvec2rotmat(extr.qvec))
         T = np.array(extr.tvec)
-
         if intr.model=="SIMPLE_PINHOLE":
             focal_length_x = intr.params[0]
             FovY = focal2fov(focal_length_x, height)

examples/requirements.txt

@@ -20,3 +20,5 @@ tensorly
 pyyaml
 matplotlib
 git+https://github.com/rahul-goel/fused-ssim@1272e21a282342e89537159e4bad508b19b34157
+
+natsort

examples/simple_trainer.py

@@ -1356,9 +1356,9 @@ def main(local_rank: int, world_rank, world_size: int, cfg: Config):
         for k in runner.splats.keys():
             runner.splats[k].data = torch.cat([ckpt["splats"][k] for ckpt in ckpts])
         step = ckpts[0]["step"]
-        # runner.save_params_into_ply_file()
-        # runner.eval(step=step)
-        # runner.render_traj(step=step)
+        runner.save_params_into_ply_file()
+        runner.eval(step=step)
+        runner.render_traj(step=step)
         if cfg.compression is not None:
             if cfg.compression == "entropy_coding":
                 runner.load_entropy_model_from_ckpt(ckpts[0], cfg.entropy_model_type)

gsplat/compression/__init__.py

@@ -1,3 +1,4 @@
 from .png_compression import PngCompression
 from .entropy_coding_compression import EntropyCodingCompression
 from .hevc_compression import HevcCompression
+from .stg_compression import STGPngCompression

mpeg_gsc_utils/__init__.py

@@ -0,0 +1,38 @@
+# Preparation for dynamic splats
+
+## Dataset file structure
+**MPEG Multiview Video Dataset**
+Please organize the data structure as follows:
+
+data/
+├── GSC
+│   ├── scene1/
+│   │   ├── colmap
+│   │   ├── mp4
+│   │   ├── png
+│   │   ├── yuv
+│   │   │   ├── v00_texture_1920x1080_yuv420p10le.yuv
+│   │   │   ├── v01_texture_1920x1080_yuv420p10le.yuv
+│   │   │   ├── ...
+│   ├── scene1/
+│   │   ├── colmap
+│   │   ├── mp4
+│   │   ├── png
+│   │   ├── yuv
+│   │   │   ├── v00_texture_1920x1080_yuv420p10le.yuv
+│   │   │   ├── v01_texture_1920x1080_yuv420p10le.yuv
+│   │   │   ├── ...
+
+## Operations
+1. Convert multiview videos in "yuv" format to "mp4" format and "png" format
+```bash
+python mpeg_gsc_utils/multiview_video_preprocess/video_preprocess.py
+```
+2. Obtain the camera intrinsic and extrinsic parameters and save as "poses_bds.npy" file
+```bash
+python mpeg_gsc_utils/multiview_video_preprocess/gen_poses_bds.py
+```
+3. Run colmap frame by frame to get per-frame SfM point clouds 
+```bash
+python mpeg_gsc_utils/multiview_video_preprocess/run_per_frame_colmap.py
+```