Merge pull request #6 from ori-drs/et-tro

Silvr 2.0

Author: YifuTao

.docker/.env

@@ -1,2 +1,9 @@
 UID=1000
-GID=1000
+GID=1000
+USERNAME=docker_dev # Set your username for the container
+
+HOST_DATA_DIR=~/data # folder on the host machine which you can attach to the container
+
+DOCKER_HOME_DIR=/home/${USERNAME} # Home directory inside the container
+DOCKER_WORK_DIR=${DOCKER_HOME_DIR}/silvr # Path to the main codebase inside the docker container
+DOCKER_DATA_DIR=${DOCKER_HOME_DIR}/data # folder inside the docker container of the attached folder HOST_DATA_DIR

.docker/Dockerfile

@@ -33,24 +33,23 @@ RUN python -m pip install --no-cache-dir git+https://github.com/NVlabs/tiny-cuda
 
 ARG GID
 ARG UID
-ENV UNAME=docker_dev
-RUN addgroup --gid $GID $UNAME
-RUN adduser --disabled-password --gecos '' --uid $UID --gid $GID $UNAME
+ARG USERNAME
+RUN addgroup --gid $GID $USERNAME
+RUN adduser --disabled-password --gecos '' --uid $UID --gid $GID $USERNAME
 
-ARG SILVR_DIR=/home/docker_dev/silvr
-WORKDIR ${SILVR_DIR}
+ARG DOCKER_HOME_DIR
+ARG DOCKER_WORK_DIR
+WORKDIR ${DOCKER_WORK_DIR}
 
-COPY ./requirements.txt ${SILVR_DIR}/requirements.txt
+COPY ./requirements.txt ${DOCKER_WORK_DIR}/requirements.txt
 RUN pip install -r requirements.txt
-COPY ./silvr/ ${SILVR_DIR}/silvr
-COPY ./pyproject.toml ${SILVR_DIR}/pyproject.toml
-COPY ./scripts/ ${SILVR_DIR}/scripts
-COPY ./config/ ${SILVR_DIR}/config
+COPY ./silvr/ ${DOCKER_WORK_DIR}/silvr
+COPY ./pyproject.toml ${DOCKER_WORK_DIR}/pyproject.toml
 RUN pip install -e .
 # Make the outputs of the container match the host
 
-RUN chown -R ${UID}:${GID} ${SILVR_DIR}/*
-USER ${UNAME}
+RUN chown -R ${UID}:${GID} ${DOCKER_HOME_DIR}/*
+USER ${USERNAME}
 RUN echo "PS1='${debian_chroot:+($debian_chroot)}\[\033[01;33m\]\u@docker-\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '" >> ~/.bashrc
 
 CMD ["/bin/bash"]

.docker/docker_compose.yaml

@@ -7,15 +7,15 @@ services:
       args:
         - UID=${UID}
         - GID=${GID}
+        - USERNAME=${USERNAME}
+        - DOCKER_HOME_DIR=${DOCKER_HOME_DIR}
+        - DOCKER_WORK_DIR=${DOCKER_WORK_DIR}
     environment:
       - NVIDIA_VISIBLE_DEVICES=all
       - NVIDIA_DRIVER_CAPABILITIES=all
     runtime: nvidia
     network_mode: "host"
     tty: true
-    # volumes:
-        # - /home/yifu/data:/home/docker_dev/data # mount the data folder
-        # - /home/yifu/workspace/silvr_public:/home/docker_dev/silvr # mount silvr folder to access the outputs and update the code
-        # - /home/yifu/.netrc:/home/docker_dev/.netrc # for wandb authentication
-
+    volumes:
+        - ${HOST_DATA_DIR}:${DOCKER_DATA_DIR}
 

README.md

@@ -16,6 +16,7 @@ Build a docker image to install dependencies and run SiLVR.
 ```
 docker compose -f .docker/docker_compose.yaml run --build silvr
 ```
+Note: You can use your own `CUDA_ARCHITECTURES` in the Dockerfile to make the `tinycudann` build quicker.
 ### Manual Installation
 You can also install SiLVR to your system manually.
 ```
@@ -33,6 +34,15 @@ pip install -e .
 
 
 ## Running
+### T-RO 25 Results
+```
+python scripts/data_downloader.py
+
+python scripts/main.py --config config/2024-03-13-roq-01-unc.yaml
+
+python scripts/main.py --config config/2024-bodleian-01+02-unc.yaml
+```
+### ICRA 24 Results
 Download sample data from [Hugging face](https://huggingface.co/datasets/ori-drs/silvr_data/tree/main), setup the [config file](./scripts/config_train.yaml), and then run the training script.
 ```bash
 python scripts/data_downloader.py
@@ -54,10 +64,10 @@ pre-commit install
 ## Citation
 If you found this software package useful, please consider citing our paper as 
 ```bibtex
-@inproceedings{tao2024silvr,
-  title = {SiLVR: Scalable Lidar-Visual Reconstruction with Neural Radiance Fields for Robotic Inspection},
-  author = {Tao, Yifu and Bhalgat, Yash and Fu, Lanke Frank Tarimo and Mattamala, Matias and Chebrolu, Nived and Fallon, Maurice},
-  booktitle = {IEEE International Conference on Robotics and Automation (ICRA)},
-  year = {2024},
+@article{tao2025silvr,
+  title={SiLVR: Scalable Lidar-Visual Radiance Field Reconstruction with Uncertainty Quantification},
+  author={Tao, Yifu and Fallon, Maurice},
+  journal={IEEE Transactions on Robotics}, 
+  year={2025},
 }
 ```

config/2024-03-13-roq-01-unc.yaml

@@ -0,0 +1,53 @@
+base:
+  method: "bayes-lidar-normal-nerfacto"
+  data: "/home/docker_dev/silvr/data/2024-03-13-roq-01-tro/transforms_colmap_scaled_lidar.json" # This will be skipped if run_submap is true
+  vis: "wandb"
+  max_train_images: 500
+  max_eval_images: 30
+  cam_optimiser_mode: "off"
+  cam_opt_lf: 0.0006
+  max_num_iterations: 30001
+  steps_per_eval_image: 2000
+  steps_per_eval_all_images: 70000
+  output_dir: "/home/docker_dev/silvr/outputs"
+
+lidar_nerf:
+  lidar_depth_loss_type: "DS_NERF_NEW"
+  is_euclidean_depth: false
+  depth_loss_mult: 10
+  normal_loss_mult: 0.001
+  depth_sigma: 0.001
+  should_decay_sigma: true
+  starting_depth_sigma: 0.01
+  sigma_decay_rate: 0.99985
+  save_pcd: false
+
+post_process:
+  save_folder_name: "post_process"
+  compute_nvs_metrics: false
+  compute_uncertainty: true
+  unc_iterations: 1000
+  unc_grid_lod: 8
+  render_uncertainty: true
+  render_max_uncertainty_point_legacy: 1
+  render_downscale: 1
+  render_cam_path_with_submap: false
+  camera_path_file: ""
+  camera_path_model_folder_path: ""
+  export_cloud: true
+  export_cloud_suffix: ""
+  cloud_max_uncertainty_ray: 1
+  export_num_points: 1000000
+  evaluate_cloud: false
+  ground_truth_3d_map_path: "/home/docker_dev/silvr/data/2024-03-13-roq-01/gt_5cm_lidar_frame.pcd"
+  T_gt_nerf_path: ""
+
+only_post_process:
+  turn_on: false # if turned on, training will be skipped
+  output_folders:
+    # - "/home/yifu/workspace/silvr/outputs/hbac_maths_19/bayes-lidar-normal-nerfacto/2024-07-14_132335"
+
+submap:
+  run_submap: true
+  data_main_folder: "/home/docker_dev/silvr/data/2024-03-13-roq-01-tro"
+  submap_folder: "/home/docker_dev/silvr/data/2024-03-13-roq-01-tro/submaps_visibility"

config/2024-bodleian-01+02-unc.yaml

@@ -0,0 +1,53 @@
+base:
+  method: "bayes-lidar-normal-nerfacto"
+  data: "/home/docker_dev/silvr/data/2024-bodleian-01+02-tro/transforms_colmap_scaled_lidar_no_cam2_bod_1.json" # This will be skipped if run_submap is true
+  vis: "wandb"
+  max_train_images: 500
+  max_eval_images: 30
+  cam_optimiser_mode: "off"
+  cam_opt_lf: 0.0006
+  max_num_iterations: 30001
+  steps_per_eval_image: 2000
+  steps_per_eval_all_images: 70000
+  output_dir: "/home/docker_dev/silvr/outputs"
+
+lidar_nerf:
+  lidar_depth_loss_type: "DS_NERF_NEW"
+  is_euclidean_depth: false
+  depth_loss_mult: 10
+  normal_loss_mult: 0.001
+  depth_sigma: 0.001
+  should_decay_sigma: true
+  starting_depth_sigma: 0.01
+  sigma_decay_rate: 0.99985
+  save_pcd: false
+
+post_process:
+  save_folder_name: "post_process"
+  compute_nvs_metrics: false
+  compute_uncertainty: true
+  unc_iterations: 2000
+  unc_grid_lod: 8
+  render_uncertainty: true
+  render_max_uncertainty_point_legacy: 1
+  render_downscale: 1
+  render_cam_path_with_submap: false
+  camera_path_file: ""
+  camera_path_model_folder_path: ""
+  export_cloud: true
+  export_cloud_suffix: ""
+  cloud_max_uncertainty_ray: 1
+  export_num_points: 1000000
+  evaluate_cloud: false
+  ground_truth_3d_map_path: ""
+  T_gt_nerf_path: ""
+
+only_post_process:
+  turn_on: false # if turned on, training will be skipped
+  output_folders:
+    # - "/home/yifu/workspace/silvr/outputs/hbac_maths_19/bayes-lidar-normal-nerfacto/2024-07-14_132335"
+
+submap:
+  run_submap: true
+  data_main_folder: "/home/docker_dev/silvr/data/2024-bodleian-01+02-tro"
+  submap_folder: "/home/docker_dev/silvr/data/2024-bodleian-01+02-tro/submaps_visibility"

pyproject.toml

@@ -18,8 +18,11 @@ dependencies = {file = ["requirements.txt"]}
 # ndp = "silvr.test:main"
 
 [project.entry-points.'nerfstudio.method_configs']
-lidar_nerfacto = 'silvr.silvr_config:Lidar_nerfacto'
 lidar_depth_nerfacto = 'silvr.silvr_config:Lidar_depth_nerfacto'
+lidar_normal_nerfacto = 'silvr.silvr_config:Lidar_normal_nerfacto'
+bayes_nerfacto = 'silvr.silvr_config:Bayes_nerfacto'
+bayes_lidar_normal_nerfacto = 'silvr.silvr_config:Bayes_Lidar_normal_nerfacto'
+bayes_lidar_normal_nerfacto_big = 'silvr.silvr_config:Bayes_Lidar_normal_nerfacto_big'
 
 [project.optional-dependencies]
 style = [

requirements.txt

@@ -4,5 +4,7 @@ gsplat==0.1.12
 pre-commit==3.0.0
 ruff==0.11.4
 tyro==0.6.6
+pytest==8.3.3
 wandb==0.13.3
 huggingface_hub==0.30.2
+hf_xet==1.2.0

scripts/cloud_cropping.py

@@ -0,0 +1,126 @@
+from pathlib import Path
+from typing import Optional
+
+import numpy as np
+import open3d as o3d
+
+from silvr.utils.eval import compute_p2p_distance, get_recon_metrics, save_error_cloud
+
+
+class EvalCloudCropper:
+    def __init__(
+        self,
+        input_cloud: np.ndarray,
+        gt_cloud: np.ndarray,
+        output_folder: str,
+        input_crop_voxel_grid: Optional[o3d.geometry.VoxelGrid] = None,
+        gt_crop_voxel_grid: Optional[o3d.geometry.VoxelGrid] = None,
+        voxel_size: float = 0.5,
+    ):
+        assert input_cloud.shape[1] == 3 and gt_cloud.shape[1] == 3  # N x 3
+        self.input_cloud = input_cloud
+        self.gt_cloud = gt_cloud
+        self.output_folder = Path(output_folder)
+        self.output_folder.mkdir(parents=True, exist_ok=True)
+        self.voxel_size = voxel_size
+
+    def compute_crop_cloud_proposal(self, max_error: float = 1.0):
+        lidar_error_cloud = self.compute_high_error_cloud(self.input_cloud, self.gt_cloud, max_error=max_error)
+        lidar_error_cloud.paint_uniform_color([1, 0, 0])
+        o3d.io.write_point_cloud(str(self.output_folder / "lidar_error_cloud.pcd"), lidar_error_cloud)
+        gt_error_cloud = self.compute_high_error_cloud(self.gt_cloud, self.input_cloud, max_error=max_error)
+        gt_error_cloud.paint_uniform_color([1, 0, 0])
+        o3d.io.write_point_cloud(str(self.output_folder / "gt_error_cloud.pcd"), gt_error_cloud)
+        return lidar_error_cloud, gt_error_cloud
+
+    def compute_high_error_cloud(self, input_cloud: np.ndarray, gt_cloud: np.ndarray, max_error: float = 1.0):
+        input_error_map = compute_p2p_distance(input_cloud, gt_cloud)
+        input_error_mask = input_error_map > max_error
+        input_error_cloud = input_cloud[input_error_mask]
+        input_error_cloud_o3d = o3d.geometry.PointCloud()
+        input_error_cloud_o3d.points = o3d.utility.Vector3dVector(input_error_cloud)
+        # lidar_error_1m_o3d.remove_statistical_outlier(nb_neighbors=20, std_ratio=2.0)
+        return input_error_cloud_o3d
+
+    def compute_crop_voxel_grid(self, cloud: np.ndarray, voxel_size: float = 0.5):
+        if isinstance(cloud, np.ndarray):
+            cloud = o3d.geometry.PointCloud(o3d.utility.Vector3dVector(cloud))
+        assert isinstance(cloud, o3d.geometry.PointCloud)
+        voxel_grid = o3d.geometry.VoxelGrid.create_from_point_cloud(cloud, voxel_size)
+        return voxel_grid
+
+    def crop_cloud_by_voxel_grid(self, cloud: np.ndarray, voxel_grid: o3d.geometry.VoxelGrid):
+        def points_in_voxels(points, voxel_grid):
+            queries = o3d.utility.Vector3dVector(points)
+            return voxel_grid.check_if_included(queries)
+
+        if isinstance(cloud, np.ndarray):
+            cloud = o3d.geometry.PointCloud(o3d.utility.Vector3dVector(cloud))
+        assert isinstance(cloud, o3d.geometry.PointCloud)
+        points_to_crop = points_in_voxels(cloud.points, voxel_grid)
+        points_to_keep = [not x for x in points_to_crop]
+        remaining_cloud = cloud.select_by_index(np.where(points_to_keep)[0])
+        return remaining_cloud
+
+
+def convert_voxel_grid_to_point_cloud(voxel_grid):
+    points = []
+    for voxel in voxel_grid.get_voxels():
+        voxel_centre = voxel_grid.get_voxel_center_coordinate(voxel.grid_index)
+        points.append(voxel_centre)
+    point_cloud = o3d.geometry.PointCloud()
+    point_cloud.points = o3d.utility.Vector3dVector(points)
+    return point_cloud
+
+
+if __name__ == "__main__":
+    folder_path = Path("/home/yifu/workspace/T-RO_2025/Observatory_quater")
+    lidar_map_path = folder_path / "ROQ_lidar_map_cleaned.pcd"
+    leica_map_path = folder_path / "roq_5cm.pcd"
+
+    T_leica_lidar = np.loadtxt(str(folder_path / "T_RTC_vilens.txt"))
+
+    lidar_map = o3d.io.read_point_cloud(str(lidar_map_path))
+    # lidar_map.remove_statistical_outlier(nb_neighbors=20, std_ratio=2.0)
+    lidar_map.transform(T_leica_lidar)
+    lidar_map = np.array(lidar_map.points)
+    leica_map = o3d.io.read_point_cloud(str(leica_map_path))
+    leica_map = np.array(leica_map.points)
+
+    output_folder_path = folder_path / "test"
+    eval_cloud_cropper = EvalCloudCropper(lidar_map, leica_map, output_folder_path)
+    create_proposed_error_cloud = True
+    crop_cloud = True
+    evaluation = True
+
+    lidar_error_cloud_path = output_folder_path / "lidar_error_cloud.pcd"
+    gt_error_cloud_path = output_folder_path / "gt_error_cloud.pcd"
+    cropped_lidar_cloud_save_path = output_folder_path / "lidar_cropped.pcd"
+    cropped_gt_cloud_save_path = output_folder_path / "gt_cropped.pcd"
+
+    if create_proposed_error_cloud:
+        lidar_error_cloud, gt_error_cloud = eval_cloud_cropper.compute_crop_cloud_proposal(max_error=0.2)
+        o3d.io.write_point_cloud(str(lidar_error_cloud_path), lidar_error_cloud)
+        o3d.io.write_point_cloud(str(gt_error_cloud_path), gt_error_cloud)
+
+    # load error cloud and convert to voxel grid, then crop the map
+    if crop_cloud:
+        lidar_error_cloud = o3d.io.read_point_cloud(str(lidar_error_cloud_path))
+        lidar_crop_voxel_grid = eval_cloud_cropper.compute_crop_voxel_grid(lidar_error_cloud)
+        lidar_cropped_cloud = eval_cloud_cropper.crop_cloud_by_voxel_grid(lidar_map, lidar_crop_voxel_grid)
+        o3d.io.write_point_cloud(str(cropped_lidar_cloud_save_path), lidar_cropped_cloud)
+
+        gt_error_cloud = o3d.io.read_point_cloud(str(gt_error_cloud_path))
+        gt_crop_voxel_grid = eval_cloud_cropper.compute_crop_voxel_grid(gt_error_cloud)
+        gt_cropped_cloud = eval_cloud_cropper.crop_cloud_by_voxel_grid(leica_map, gt_crop_voxel_grid)
+        o3d.io.write_point_cloud(str(cropped_gt_cloud_save_path), gt_cropped_cloud)
+
+    # apply cropping
+    if evaluation:
+        lidar_cropped_cloud_np = np.array(o3d.io.read_point_cloud(str(cropped_lidar_cloud_save_path)).points)
+        gt_cropped_cloud_np = np.array(o3d.io.read_point_cloud(str(cropped_gt_cloud_save_path)).points)
+        print(get_recon_metrics(lidar_cropped_cloud_np, gt_cropped_cloud_np))
+        lidar_error_cmap_cloud_save_path = output_folder_path / "lidar_error_cmap_cloud.ply"
+        save_error_cloud(lidar_cropped_cloud_np, gt_cropped_cloud_np, lidar_error_cmap_cloud_save_path)
+        gt_error_cmap_cloud_save_path = output_folder_path / "gt_error_cmap_cloud.ply"
+        save_error_cloud(gt_cropped_cloud_np, lidar_cropped_cloud_np, gt_error_cmap_cloud_save_path)