add jupyter notebooks and fix codes

Author: tiffanyluong

src/b3d/unity/Meshtest.ipynb

@@ -0,0 +1,201 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from pathlib import Path\n",
+    "import b3d\n",
+    "from b3d.io.feature_track_data import FeatureTrackData\n",
+    "from b3d.io.segmented_video_input import SegmentedVideoInput\n",
+    "from b3d.unity.os_codes.generate_visualization import create_video, create_rgb_image, create_feature_track_video, create_segmented_video_input_video"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "root_dir = str(b3d.get_shared_large() / 'unity').strip()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def create_teaser(data_path, video_path):\n",
+    "    feature_track_data = FeatureTrackData.load(data_path)\n",
+    "    print(f\"creating {video_path}\")\n",
+    "    create_video(feature_track_data.rgbd_images[:,:,:,:3], create_rgb_image, output_path=video_path, label=None, res=None, fps=10, slow=1, source_fps=30)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def add_mp4_teaser(root_dir):\n",
+    "    root_path = Path(root_dir)\n",
+    "    for scene_folder in root_path.iterdir():\n",
+    "        for data_folder in scene_folder.iterdir():\n",
+    "            video_path = Path(data_folder / f\"{data_folder.name}_teaser.mp4\")\n",
+    "            # # if not os.path.exists(video_path):\n",
+    "            feature_track_folder = Path(data_folder / 'feature_track_data')\n",
+    "            for files in feature_track_folder.iterdir():\n",
+    "                if files.name.endswith('_bg_800p.input.npz'):\n",
+    "                    create_teaser(files, video_path)\n",
+    "\n",
+    "# add_mp4_teaser(root_dir)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processing /home/tiffa/b3d/assets/large_data_bucket/unity/primitives/cubespheretest/feature_track_data/lit_nobg_800p.input.npz\n",
+      "Processing /home/tiffa/b3d/assets/large_data_bucket/unity/primitives/fppcube/feature_track_data/lit_nobg_800p.input.npz\n",
+      "Processing /home/tiffa/b3d/assets/large_data_bucket/unity/primitives/bouncingcube/feature_track_data/lit_bg_800p.input.npz\n",
+      "Processing /home/tiffa/b3d/assets/large_data_bucket/unity/primitives/bouncingcubecorners/feature_track_data/lit_nobg_800p.input.npz\n",
+      "Processing /home/tiffa/b3d/assets/large_data_bucket/unity/primitives/cubeball/feature_track_data/lit_nobg_800p.input.npz\n",
+      "Processing /home/tiffa/b3d/assets/large_data_bucket/unity/primitives/fppcubecorners/feature_track_data/lit_nobg_800p.input.npz\n",
+      "Processing /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/lookingatrocket/feature_track_data/lit_bg_800p.input.npz\n",
+      "Processing /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/blockdisappearing/feature_track_data/lit_bg_800p.input.npz\n",
+      "Processing /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/blockdisappearing/segmented_video_input/lit_bg_800p.input.npz\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/opt/conda/envs/b3d/lib/python3.12/subprocess.py:1885: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock.\n",
+      "  self.pid = _fork_exec(\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Saved /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/blockdisappearing/segmented_video_input/lit_bg.mp4\n",
+      "Processing /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/blockmostlyoccluded/feature_track_data/lit_bg_800p.input.npz\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/opt/conda/envs/b3d/lib/python3.12/subprocess.py:1885: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock.\n",
+      "  self.pid = _fork_exec(\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Saved /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/blockmostlyoccluded/feature_track_data/lit_bg.mp4\n",
+      "Processing /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/blockmostlyoccluded/segmented_video_input/lit_bg_800p.input.npz\n",
+      "Saved /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/blockmostlyoccluded/segmented_video_input/lit_bg.mp4\n",
+      "Processing /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/rotatingaroundrocket/feature_track_data/lit_bg_800p.input.npz\n",
+      "Saved /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/rotatingaroundrocket/feature_track_data/lit_bg.mp4\n",
+      "Processing /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/ballstriking/feature_track_data/lit_bg_800p.input.npz\n",
+      "Saved /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/ballstriking/feature_track_data/lit_bg.mp4\n",
+      "Processing /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/blockfullyoccluded/feature_track_data/lit_bg_800p.input.npz\n",
+      "Saved /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/blockfullyoccluded/feature_track_data/lit_bg.mp4\n",
+      "Processing /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/blockfullyoccluded/segmented_video_input/lit_bg_800p.input.npz\n",
+      "Saved /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/blockfullyoccluded/segmented_video_input/lit_bg.mp4\n",
+      "Processing /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/blockpartiallyoccluded/feature_track_data/lit_bg_800p.input.npz\n",
+      "Saved /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/blockpartiallyoccluded/feature_track_data/lit_bg.mp4\n",
+      "Processing /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/blockpartiallyoccluded/segmented_video_input/lit_bg_800p.input.npz\n",
+      "Saved /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/blockpartiallyoccluded/segmented_video_input/lit_bg.mp4\n",
+      "Processing /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/blockswap/feature_track_data/lit_bg_800p.input.npz\n",
+      "Saved /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/blockswap/feature_track_data/lit_bg.mp4\n",
+      "Processing /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/blockswap/segmented_video_input/lit_bg_800p.input.npz\n",
+      "Saved /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/blockswap/segmented_video_input/lit_bg.mp4\n",
+      "Processing /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/blocksfalling/feature_track_data/lit_bg_800p.input.npz\n",
+      "Saved /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/blocksfalling/feature_track_data/lit_bg.mp4\n",
+      "Processing /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/fpprocket/feature_track_data/lit_bg_800p.input.npz\n",
+      "Saved /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/fpprocket/feature_track_data/lit_bg.mp4\n",
+      "Processing /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/fpprocket/feature_track_data/lit_nobg_800p.input.npz\n",
+      "Saved /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/fpprocket/feature_track_data/lit_nobg.mp4\n",
+      "Processing /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/rocketrotating/feature_track_data/lit_bg_800p.input.npz\n",
+      "Saved /home/tiffa/b3d/assets/large_data_bucket/unity/toyroom/rocketrotating/feature_track_data/lit_bg.mp4\n"
+     ]
+    }
+   ],
+   "source": [
+    "def create_feature_track_video_in_folder(data_path, folder):\n",
+    "    parts = data_path.name.split('_')\n",
+    "    light_setting = parts[0]\n",
+    "    bg_setting = parts[1]\n",
+    "    video_path = Path(folder / f\"{light_setting}_{bg_setting}.mp4\")\n",
+    "    if not os.path.exists(video_path):\n",
+    "        feature_track_data = FeatureTrackData.load(data_path)\n",
+    "        create_feature_track_video(feature_track_data, output_path=video_path, res=None, fps=10, slow=1, source_fps=30)\n",
+    "\n",
+    "def create_segmentation_video_in_folder(data_path, folder):\n",
+    "    parts = data_path.name.split('_')\n",
+    "    light_setting = parts[0]\n",
+    "    bg_setting = parts[1]\n",
+    "    video_path = Path(folder / f\"{light_setting}_{bg_setting}.mp4\")\n",
+    "    if not os.path.exists(video_path):\n",
+    "        segmentation_data = SegmentedVideoInput.load(data_path)\n",
+    "        create_segmented_video_input_video(segmentation_data, output_path=video_path, res=None, fps=10, slow=1, source_fps=30)\n",
+    "\n",
+    "def add_mp4_settings(root_dir):\n",
+    "    root_path = Path(root_dir)\n",
+    "    for scene_folder in root_path.iterdir():\n",
+    "        for data_folder in scene_folder.iterdir():\n",
+    "\n",
+    "            feature_track_folder = Path(data_folder / 'feature_track_data')\n",
+    "            if os.path.exists(feature_track_folder):\n",
+    "                for files in feature_track_folder.iterdir():\n",
+    "                    if files.name.endswith('800p.input.npz'):\n",
+    "                        print(f\"Processing {files}\")\n",
+    "                        create_feature_track_video_in_folder(files, feature_track_folder)\n",
+    "                        \n",
+    "\n",
+    "            segmented_folder = Path(data_folder / 'segmented_video_input')\n",
+    "            if os.path.exists(segmented_folder):\n",
+    "                for files in segmented_folder.iterdir():\n",
+    "                    if files.name.endswith('800p.input.npz'):\n",
+    "                        print(f\"Processing {files}\")\n",
+    "                        create_segmentation_video_in_folder(files, segmented_folder)\n",
+    "                        # return\n",
+    "\n",
+    "add_mp4_settings(root_dir)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "b3d",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/b3d/unity/changing_img_format.ipynb

@@ -0,0 +1,63 @@
+import os
+from pathlib import Path
+import b3d
+from b3d.io.feature_track_data import FeatureTrackData
+from b3d.unity.generate_visualization import create_video, create_rgb_image
+import json
+
+def get_tags(data_name):
+    tags_dict = {
+        1: 'dynamic camera',
+        2: 'dynamic objects',
+        3: 'panning',
+        4: 'self-occlusion',
+        5: 'occlusion',
+        6: 'deformable objects',
+    }
+    
+    print(f"Available tags for {data_name}:")
+    for key, value in tags_dict.items():
+        print(f"{key}: {value}")
+    
+    selected_tags = input(f"Enter the numbers corresponding to the tags for {data_name}, separated by commas: ").strip()
+    if (selected_tags == ""):
+        return []
+    
+    tags = [tags_dict[int(num)] for num in selected_tags.split(",") if int(num) in tags_dict]
+    
+    return tags
+
+def process(root_dir):
+    root_path = Path(root_dir)
+    for scene_folder in root_path.iterdir():
+        for data_folder in scene_folder.iterdir():
+            print(f"Processing {data_folder.name}")
+            for data_class_content in data_folder.iterdir():
+                if (os.path.isdir(data_class_content)):
+                    data_class_folder = data_class_content
+                    for files in data_class_folder.iterdir():
+                        if files.name.endswith('.gif'):
+                            print(f"removing {files}")
+                            os.remove(files)
+                if (data_class_content.name.endswith('.json')):
+                    print(f"editing {data_class_content}")
+                    metadata_file = data_class_content
+                    combined_metadata = {"tags": []}
+                    with metadata_file.open('r') as f:
+                        metadata = json.load(f)
+                        combined_metadata["tags"].extend(metadata.get("tags", []))
+                        print(f"Current tags: {combined_metadata}")
+
+                        additional_tags = get_tags(data_folder.name)
+                        combined_metadata["tags"].extend(additional_tags)
+                        # Remove duplicates from combined_metadata
+                        combined_metadata["tags"] = list(set(combined_metadata["tags"]))
+
+                    # Save the combined metadata at the data_folder level
+                    with metadata_file.open('w') as f:
+                        json.dump(combined_metadata, f, indent=4)
+                        print(f"Edited tags: {combined_metadata}")
+        
+if __name__ == "__main__":
+    root_dir = str(b3d.get_shared_large() / 'unity').strip()
+    process(root_dir)

src/b3d/unity/data_processingtest.ipynb

@@ -0,0 +1,84 @@
+import os
+import json
+from pathlib import Path
+import shutil
+import b3d
+
+def get_tags(data_name):
+    tags_dict = {
+        1: 'dynamic camera',
+        2: 'dynamic objects',
+        3: 'panning',
+        4: 'deformable objects',
+        5: 'self-occlusion',
+        6: 'occlusion',
+    }
+    
+    print(f"Available tags for {data_name}:")
+    for key, value in tags_dict.items():
+        print(f"{key}: {value}")
+    
+    selected_tags = input(f"Enter the numbers corresponding to the tags for {data_name}, separated by commas: ").strip()
+    tags = [tags_dict[int(num)] for num in selected_tags.split(",") if int(num) in tags_dict]
+    
+    return tags
+
+def get_additional_data(data_name):
+    options = {
+        1: 'None',
+        2: 'mesh'
+    }
+    
+    print(f"Available additional data for {data_name}:")
+    for key, value in options.items():
+        print(f"{key}: {value}")
+    
+    selected_data = input(f"Enter the number corresponding to the additional data for {data_name}: ").strip()
+    additional_data = options.get(int(selected_data), 'None')
+    
+    return [] if additional_data == 'None' else [additional_data]
+
+def consolidate_metadata(root):
+    root_path = Path(root)
+    for scene_folder in root_path.iterdir():
+        if scene_folder.is_dir():
+            for data_folder in scene_folder.iterdir():
+                if data_folder.is_dir():
+                    combined_metadata = {"tags": [], "additional_data": []}
+                    
+                    for data_class_folder in data_folder.iterdir():
+                        if data_class_folder.is_dir():
+                            metadata_file = data_class_folder / "metadata.json"
+                            if metadata_file.exists():
+                                with metadata_file.open('r') as f:
+                                    metadata = json.load(f)
+                                    combined_metadata["tags"].extend(metadata.get("tags", []))
+                                    combined_metadata["additional_data"].extend(metadata.get("additional_data", []))
+                                
+                                # Remove the metadata file from the data_class folder
+                                metadata_file.unlink()
+                    
+                    # Request user input for tags and additional data
+                    data_name = data_folder.name
+                    tags = get_tags(data_name)
+                    additional_data = get_additional_data(data_name)
+
+                    # Combine user input with existing metadata
+                    combined_metadata["tags"].extend(tags)
+                    combined_metadata["additional_data"].extend(additional_data)
+
+                    # Remove duplicates from combined_metadata
+                    combined_metadata["tags"] = list(set(combined_metadata["tags"]))
+                    combined_metadata["additional_data"] = list(set(combined_metadata["additional_data"]))
+
+                    # Save the combined metadata at the data_folder level
+                    new_metadata_file = data_folder / "metadata.json"
+                    with new_metadata_file.open('w') as f:
+                        json.dump(combined_metadata, f, indent=4)
+
+                    print(f"Consolidated metadata for {data_folder.name}")
+
+if __name__ == "__main__":
+    root_dir = str(b3d.get_shared_large() / 'unity').strip()
+    consolidate_metadata(root_dir)
+    print("Metadata consolidation complete.")