feat: silvr 2.0

Author: YifuTao

.docker/.env

@@ -1,2 +1,9 @@
 UID=1000
-GID=1000
+GID=1000
+USERNAME=docker_dev # Set your username for the container
+
+HOST_DATA_DIR=~/data # folder on the host machine which you can attach to the container
+
+DOCKER_HOME_DIR=/home/${USERNAME} # Home directory inside the container
+DOCKER_WORK_DIR=${DOCKER_HOME_DIR}/silvr # Path to the main codebase inside the docker container
+DOCKER_DATA_DIR=${DOCKER_HOME_DIR}/data # folder inside the docker container of the attached folder HOST_DATA_DIR

.docker/Dockerfile

@@ -33,24 +33,23 @@ RUN python -m pip install --no-cache-dir git+https://github.com/NVlabs/tiny-cuda
 
 ARG GID
 ARG UID
-ENV UNAME=docker_dev
-RUN addgroup --gid $GID $UNAME
-RUN adduser --disabled-password --gecos '' --uid $UID --gid $GID $UNAME
+ARG USERNAME
+RUN addgroup --gid $GID $USERNAME
+RUN adduser --disabled-password --gecos '' --uid $UID --gid $GID $USERNAME
 
-ARG SILVR_DIR=/home/docker_dev/silvr
-WORKDIR ${SILVR_DIR}
+ARG DOCKER_HOME_DIR
+ARG DOCKER_WORK_DIR
+WORKDIR ${DOCKER_WORK_DIR}
 
-COPY ./requirements.txt ${SILVR_DIR}/requirements.txt
+COPY ./requirements.txt ${DOCKER_WORK_DIR}/requirements.txt
 RUN pip install -r requirements.txt
-COPY ./silvr/ ${SILVR_DIR}/silvr
-COPY ./pyproject.toml ${SILVR_DIR}/pyproject.toml
-COPY ./scripts/ ${SILVR_DIR}/scripts
-COPY ./config/ ${SILVR_DIR}/config
+COPY ./silvr/ ${DOCKER_WORK_DIR}/silvr
+COPY ./pyproject.toml ${DOCKER_WORK_DIR}/pyproject.toml
 RUN pip install -e .
 # Make the outputs of the container match the host
 
-RUN chown -R ${UID}:${GID} ${SILVR_DIR}/*
-USER ${UNAME}
+RUN chown -R ${UID}:${GID} ${DOCKER_HOME_DIR}/*
+USER ${USERNAME}
 RUN echo "PS1='${debian_chroot:+($debian_chroot)}\[\033[01;33m\]\u@docker-\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '" >> ~/.bashrc
 
 CMD ["/bin/bash"]

.docker/docker_compose.yaml

@@ -7,15 +7,15 @@ services:
       args:
         - UID=${UID}
         - GID=${GID}
+        - USERNAME=${USERNAME}
+        - DOCKER_HOME_DIR=${DOCKER_HOME_DIR}
+        - DOCKER_WORK_DIR=${DOCKER_WORK_DIR}
     environment:
       - NVIDIA_VISIBLE_DEVICES=all
       - NVIDIA_DRIVER_CAPABILITIES=all
     runtime: nvidia
     network_mode: "host"
     tty: true
-    # volumes:
-        # - /home/yifu/data:/home/docker_dev/data # mount the data folder
-        # - /home/yifu/workspace/silvr_public:/home/docker_dev/silvr # mount silvr folder to access the outputs and update the code
-        # - /home/yifu/.netrc:/home/docker_dev/.netrc # for wandb authentication
-
+    volumes:
+        - ${HOST_DATA_DIR}:${DOCKER_DATA_DIR}
 

README.md

@@ -16,6 +16,7 @@ Build a docker image to install dependencies and run SiLVR.
 ```
 docker compose -f .docker/docker_compose.yaml run --build silvr
 ```
+Note: You can use your own `CUDA_ARCHITECTURES` in the Dockerfile to make the `tinycudann` build quicker.
 ### Manual Installation
 You can also install SiLVR to your system manually.
 ```
@@ -33,6 +34,18 @@ pip install -e .
 
 
 ## Running
+### T-RO 25 Results
+```
+# download data
+python scripts/data_downloader.py
+
+# Sequence 1: ROQ
+python scripts/main.py --config config/2024-03-13-roq-01-unc.yaml
+
+# Sequence 2: Bodleian 
+python scripts/main.py --config config/2024-bodleian-01+02-tro-unc.yaml
+```
+### ICRA 24 Results
 Download sample data from [Hugging face](https://huggingface.co/datasets/ori-drs/silvr_data/tree/main), setup the [config file](./scripts/config_train.yaml), and then run the training script.
 ```bash
 python scripts/data_downloader.py
@@ -54,10 +67,10 @@ pre-commit install
 ## Citation
 If you found this software package useful, please consider citing our paper as 
 ```bibtex
-@inproceedings{tao2024silvr,
-  title = {SiLVR: Scalable Lidar-Visual Reconstruction with Neural Radiance Fields for Robotic Inspection},
-  author = {Tao, Yifu and Bhalgat, Yash and Fu, Lanke Frank Tarimo and Mattamala, Matias and Chebrolu, Nived and Fallon, Maurice},
-  booktitle = {IEEE International Conference on Robotics and Automation (ICRA)},
-  year = {2024},
+@article{tao2025silvr,
+  title={SiLVR: Scalable Lidar-Visual Radiance Field Reconstruction with Uncertainty Quantification},
+  author={Tao, Yifu and Fallon, Maurice},
+  journal={IEEE Transactions on Robotics}, 
+  year={2025},
 }
 ```

config/2024-03-13-roq-01-unc.yaml

@@ -0,0 +1,53 @@
+base:
+  method: "bayes-lidar-normal-nerfacto"
+  data: "/home/docker_dev/silvr/data/2024-03-13-roq-01-tro/transforms_colmap_scaled_lidar.json" # This will be skipped if run_submap is true
+  vis: "wandb"
+  max_train_images: 200
+  max_eval_images: 30
+  cam_optimiser_mode: "off"
+  cam_opt_lf: 0.0006
+  max_num_iterations: 30001
+  steps_per_eval_image: 2000
+  steps_per_eval_all_images: 70000
+  output_dir: "/home/docker_dev/silvr/outputs"
+
+lidar_nerf:
+  lidar_depth_loss_type: "DS_NERF_NEW"
+  is_euclidean_depth: false
+  depth_loss_mult: 10
+  normal_loss_mult: 0.001
+  depth_sigma: 0.001
+  should_decay_sigma: true
+  starting_depth_sigma: 0.01
+  sigma_decay_rate: 0.99985
+  save_pcd: false
+
+post_process:
+  save_folder_name: "post_process"
+  compute_nvs_metrics: false
+  compute_uncertainty: true
+  unc_iterations: 1000
+  unc_grid_lod: 8
+  render_uncertainty: true
+  render_max_uncertainty_point_legacy: 1
+  render_downscale: 1
+  render_cam_path_with_submap: false
+  camera_path_file: ""
+  camera_path_model_folder_path: ""
+  export_cloud: true
+  export_cloud_suffix: ""
+  cloud_max_uncertainty_ray: 1
+  export_num_points: 1000000
+  evaluate_cloud: false
+  ground_truth_3d_map_path: "/home/docker_dev/silvr/data/2024-03-13-roq-01/gt_5cm_lidar_frame.pcd"
+  T_gt_nerf_path: ""
+
+only_post_process:
+  turn_on: false # if turned on, training will be skipped
+  output_folders:
+    # - "/home/yifu/workspace/silvr/outputs/hbac_maths_19/bayes-lidar-normal-nerfacto/2024-07-14_132335"
+
+submap:
+  run_submap: true
+  data_main_folder: "/home/docker_dev/silvr/data/2024-03-13-roq-01-tro"
+  submap_folder: "/home/docker_dev/silvr/data/2024-03-13-roq-01-tro/submaps_visibility"

config/2024-bodleian-01+02-tro-unc.yaml

@@ -0,0 +1,53 @@
+base:
+  method: "bayes-lidar-normal-nerfacto"
+  data: "/home/docker_dev/silvr/data/2024-bodleian-01+02-tro/transforms_colmap_scaled_lidar_no_cam2_bod_1.json" # This will be skipped if run_submap is true
+  vis: "wandb"
+  max_train_images: 500
+  max_eval_images: 30
+  cam_optimiser_mode: "off"
+  cam_opt_lf: 0.0006
+  max_num_iterations: 30001
+  steps_per_eval_image: 2000
+  steps_per_eval_all_images: 70000
+  output_dir: "/home/docker_dev/silvr/outputs"
+
+lidar_nerf:
+  lidar_depth_loss_type: "DS_NERF_NEW"
+  is_euclidean_depth: false
+  depth_loss_mult: 10
+  normal_loss_mult: 0.001
+  depth_sigma: 0.001
+  should_decay_sigma: true
+  starting_depth_sigma: 0.01
+  sigma_decay_rate: 0.99985
+  save_pcd: false
+
+post_process:
+  save_folder_name: "post_process"
+  compute_nvs_metrics: false
+  compute_uncertainty: true
+  unc_iterations: 1000
+  unc_grid_lod: 8
+  render_uncertainty: true
+  render_max_uncertainty_point_legacy: 1
+  render_downscale: 1
+  render_cam_path_with_submap: false
+  camera_path_file: ""
+  camera_path_model_folder_path: ""
+  export_cloud: true
+  export_cloud_suffix: ""
+  cloud_max_uncertainty_ray: 1
+  export_num_points: 1000000
+  evaluate_cloud: false
+  ground_truth_3d_map_path: ""
+  T_gt_nerf_path: ""
+
+only_post_process:
+  turn_on: false # if turned on, training will be skipped
+  output_folders:
+    # - "/home/yifu/workspace/silvr/outputs/hbac_maths_19/bayes-lidar-normal-nerfacto/2024-07-14_132335"
+
+submap:
+  run_submap: true
+  data_main_folder: "/home/docker_dev/silvr/data/2024-bodleian-01+02-tro"
+  submap_folder: "/home/docker_dev/silvr/data/2024-bodleian-01+02-tro/submaps_visibility"

config/bodleian.yaml

@@ -0,0 +1,56 @@
+base:
+  method: "bayes-lidar-normal-nerfacto"
+  data: "/home/yifu/data/nerfstudio/hbac_maths_19/submap_2_cut.json"
+  vis: "wandb"
+  max_train_images: 200
+  max_eval_images: 30
+  cam_optimiser_mode: "off"
+  cam_opt_lf: 0.0006
+  max_num_iterations: 1001
+  steps_per_eval_image: 2000
+  steps_per_eval_all_images: 70000
+  output_dir: "/home/docker_dev/silvr/outputs"
+
+lidar_nerf:
+  lidar_depth_loss_type: "DS_NERF_NEW"
+  depth_loss_mult: 10
+  normal_loss_mult: 0.001
+  is_euclidean_depth: false
+  depth_sigma: 0.001
+  should_decay_sigma: true
+  starting_depth_sigma: 0.01
+  sigma_decay_rate: 0.99985
+  save_pcd: false
+
+post_process:
+  save_folder_name: "post_process"
+  compute_nvs_metrics: true
+  compute_uncertainty: false
+  unc_iterations: 10000
+  unc_grid_lod: 8
+  render_uncertainty: false
+  render_max_uncertainty_point_legacy: 1
+  render_downscale: 1
+  render_cam_path_with_submap: false
+  camera_path_file: ""
+  camera_path_model_folder_path: ""
+  export_cloud: true
+  export_cloud_suffix: ""
+  # cloud_max_uncertainty_point_legacy: 1
+  cloud_max_uncertainty_ray: 1
+  export_num_points: 1000000
+  evaluate_cloud: true
+  grond_truth_3d_map_path: "/home/docker_dev/oxford_spires_dataset/2024-03-13-observatory-quarter-01/outputs/recon_benchmark/gt_cloud_merged.pcd"
+  T_gt_nerf_path: "/home/docker_dev/data/hbac_maths_19/T_gt_nerf.txt"
+
+only_post_process:
+  turn_on: false # if turned on, training will be skipped
+  output_folders:
+    # - "/home/yifu/workspace/silvr/outputs/hbac_maths_19/bayes-lidar-normal-nerfacto/2024-07-14_132335"
+    - "/home/yifu/workspace/silvr/outputs/hbac_maths_19/bayes-lidar-normal-nerfacto/2024-07-14_134014"
+    - "/home/yifu/workspace/silvr/outputs/hbac_maths_19/bayes-lidar-normal-nerfacto/2024-07-14_135719"
+
+submap:
+  run_submap: true
+  data_main_folder: "/home/yifu/data/nerfstudio/hbac_maths_19"
+  submap_folder: "/home/yifu/data/nerfstudio/hbac_maths_19/submaps"

pyproject.toml

@@ -18,8 +18,11 @@ dependencies = {file = ["requirements.txt"]}
 # ndp = "silvr.test:main"
 
 [project.entry-points.'nerfstudio.method_configs']
-lidar_nerfacto = 'silvr.silvr_config:Lidar_nerfacto'
 lidar_depth_nerfacto = 'silvr.silvr_config:Lidar_depth_nerfacto'
+lidar_normal_nerfacto = 'silvr.silvr_config:Lidar_normal_nerfacto'
+bayes_nerfacto = 'silvr.silvr_config:Bayes_nerfacto'
+bayes_lidar_normal_nerfacto = 'silvr.silvr_config:Bayes_Lidar_normal_nerfacto'
+bayes_lidar_normal_nerfacto_big = 'silvr.silvr_config:Bayes_Lidar_normal_nerfacto_big'
 
 [project.optional-dependencies]
 style = [

requirements.txt

@@ -4,5 +4,7 @@ gsplat==0.1.12
 pre-commit==3.0.0
 ruff==0.11.4
 tyro==0.6.6
+pytest==8.3.3
 wandb==0.13.3
 huggingface_hub==0.30.2
+hf_xet==1.2.0