Migrate habitat-lab to habitat-sim agent-sensor-decoupling API (#2220)

aclegg3 · aihabitat · web-flow · commit a9c8df586d64 · 2026-02-20T16:11:03.000-10:00
* Migrate habitat-lab to habitat-sim agent-sensor-decoupling API Adapt habitat-lab to the new public sensor API from habitat-sim PR #2621 (agent-sensor-decoupling branch). Key changes: - Replace all private _sensors dict access with public sim.sensors property - Replace _sim._sensors[name]._sensor_object with sim.get_sensor(name).sensor_object - Replace agent._sensors with agent.sensors (live C++ subtree view) - Simplify instance_image_nav_task sensor lifecycle to use sim.remove_sensor() instead of manual multi-dict cleanup - Remove Python name-mangling (_Simulator__sensors) from debug_visualizer and its tests; use sim.sensors registry directly - Update CI workflow to build habitat-sim from the PR branch Companion to: facebookresearch/habitat-sim#2621 --------- Co-authored-by: Alexander William Clegg <alexclegg@meta.com>
diff --git a/.github/actions/install_ubuntu_deps/action.yml b/.github/actions/install_ubuntu_deps/action.yml
@@ -48,7 +48,7 @@ runs:
     uses: conda-incubator/setup-miniconda@v3.0.1
     with:
       miniconda-version: "latest"
-      python-version: "3.12"
+      python-version: "3.11"
       activate-environment: "habitat"
   - name: Install conda and dependencies
     run: |-
diff --git a/.github/workflows/install_and_test.yml b/.github/workflows/install_and_test.yml
@@ -76,15 +76,13 @@ jobs:
         #activate conda env
         export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
         conda activate habitat
-        #install habitat-sim
+        #install habitat-sim from nightly conda
+        conda install -y -c conda-forge -c aihabitat-nightly habitat-sim headless withbullet
+        #clone habitat-sim repo for test data and benchmark scripts
         if [ ! -d ./habitat-sim ]
         then
           git clone https://github.com/facebookresearch/habitat-sim.git --recursive
         fi
-        cd habitat-sim
-        pip install -r requirements.txt --progress-bar off
-        git submodule update --init --recursive --jobs 8
-        python -u setup.py install --headless --with-cuda --bullet
     - name: Download test data
       run: |
         # Disable clone protection for git lfs
@@ -182,7 +180,7 @@ jobs:
       uses: conda-incubator/setup-miniconda@v3.0.1
       with:
         miniconda-version: "latest"
-        python-version: "3.12"
+        python-version: "3.11"
         activate-environment: "non-editable-install"
     - name: Ensure non-editable mode works
       run: |-
@@ -214,7 +212,7 @@ jobs:
       uses: conda-incubator/setup-miniconda@v3.0.1
       with:
         miniconda-version: "latest"
-        python-version: "3.12"
+        python-version: "3.11"
         activate-environment: "bdist-install"
     - name: Ensure bdist install works and build sdist
       run: |-
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -22,14 +22,14 @@ repos:
         args: ['--fix=lf']
 
 -   repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
-    rev: v2.3.0
+    rev: v2.14.0
     hooks:
     -   id: pretty-format-ini
         args: [--autofix]
     -   id: pretty-format-toml
         args: [--autofix]
         additional_dependencies:
-         - toml-sort==0.21.0
+         - toml-sort==0.23.1
 
 -   repo: https://github.com/pre-commit/pygrep-hooks
     rev: v1.9.0  # Use the ref you want to point at
diff --git a/Dockerfile b/Dockerfile
@@ -38,11 +38,11 @@ RUN ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake
 RUN cmake --version
 
 # Conda environment
-RUN conda create -n habitat python=3.9 cmake=3.14.0
+RUN conda create -n habitat python=3.12 cmake=3.28
 
 # Setup habitat-sim
-RUN git clone --branch stable https://github.com/facebookresearch/habitat-sim.git
-RUN /bin/bash -c ". activate habitat; cd habitat-sim; pip install -r requirements.txt; python setup.py install --headless"
+RUN git clone --branch stable https://github.com/facebookresearch/habitat-sim.git --recursive
+RUN /bin/bash -c ". activate habitat; cd habitat-sim; pip install -r requirements.txt; HABITAT_BUILD_GUI_VIEWERS=OFF pip install . --no-build-isolation"
 
 # Install challenge specific habitat-lab
 RUN git clone --branch stable https://github.com/facebookresearch/habitat-lab.git
diff --git a/examples/hitl/rearrange_v2/rearrange_v2.py b/examples/hitl/rearrange_v2/rearrange_v2.py
@@ -622,7 +622,7 @@ def __init__(
             agent_cfg = rearrange_v2_config.agents[agent_index]
             head_sensor_substring = agent_cfg.head_sensor_substring
             agent_name = sim.agents_mgr.get_agent_name_from_index(agent_index)
-            for sensor_name, sensor in sim_agent._sensors.items():
+            for sensor_name, sensor in sim_agent.sensors.items():
                 if (
                     isinstance(sensor, VisualSensor)
                     and agent_name in sensor_name
diff --git a/examples/interactive_play.py b/examples/interactive_play.py
@@ -450,9 +450,9 @@ def update(self, env, step_result, update_idx):
             trans = mn.Matrix4.from_(
                 quat.to_matrix(), mn.Vector3(*self._free_xyz)
             )
-            env._sim._sensors[
+            env._sim.get_sensor(
                 "third_rgb"
-            ]._sensor_object.node.transformation = trans
+            ).sensor_object.node.transformation = trans
             step_result = env._sim.get_sensor_observations()
             return step_result
         return step_result
diff --git a/habitat-hitl/README.md b/habitat-hitl/README.md
@@ -33,7 +33,7 @@ Example HITL apps are configured to run at 30 steps per second (SPS). If your sy
     * The HITL framework depends on the `habitat-lab` and `habitat-baselines` packages. While these packages are in the same repository as the HITL framework, it's not strictly necessary for the HITL framework to import the packages that live in th- [Human-in-the-loop (HITL) Framework](#human-in-the-loop-hitl-framework)
 3. Install Habitat-sim [main branch](https://github.com/facebookresearch/habitat-sim).
     * [Build from source](https://github.com/facebookresearch/habitat-sim/blob/main/BUILD_FROM_SOURCE.md), or install the [conda packages](https://github.com/facebookresearch/habitat-sim#recommended-conda-packages).
-        * Be sure to include Bullet physics, e.g. `python setup.py install --bullet`.
+        * Be sure to include Bullet physics, e.g. `HABITAT_WITH_BULLET=ON pip install . --no-build-isolation` or `./build.sh --with-bullet`.
 4. Install the `habitat-hitl` package.
     * From the `habitat-lab` root directory, run `pip install -e habitat-hitl`.
 5. Download required assets for our example HITL applications (Note that the dataset downloader should be run from habitat-lab/.):
diff --git a/habitat-lab/habitat/articulated_agents/articulated_agent_base.py b/habitat-lab/habitat/articulated_agents/articulated_agent_base.py
@@ -77,7 +77,7 @@ def __init__(
         if hasattr(self.params, "cameras"):
             self._cameras = defaultdict(list)
             for camera_prefix in self.params.cameras:
-                for sensor_name in self._sim._sensors:
+                for sensor_name in self._sim.sensors:
                     if sensor_name.startswith(camera_prefix):
                         self._cameras[camera_prefix].append(sensor_name)
 
diff --git a/habitat-lab/habitat/articulated_agents/humanoids/kinematic_humanoid.py b/habitat-lab/habitat/articulated_agents/humanoids/kinematic_humanoid.py
@@ -201,7 +201,7 @@ def update(self) -> None:
             # update the cameras
             for cam_prefix, sensor_names in self._cameras.items():
                 for sensor_name in sensor_names:
-                    sens_obj = self._sim._sensors[sensor_name]._sensor_object
+                    sens_obj = self._sim.get_sensor(sensor_name).sensor_object
                     cam_info = self.params.cameras[cam_prefix]
 
                     if cam_info.attached_link_id == -1:
diff --git a/habitat-lab/habitat/articulated_agents/manipulator.py b/habitat-lab/habitat/articulated_agents/manipulator.py
@@ -71,7 +71,7 @@ def __init__(
         if hasattr(self.params, "cameras"):
             self._cameras = defaultdict(list)
             for camera_prefix in self.params.cameras:
-                for sensor_name in self._sim._sensors:
+                for sensor_name in self._sim.sensors:
                     if sensor_name.startswith(camera_prefix):
                         self._cameras[camera_prefix].append(sensor_name)
 
@@ -149,7 +149,7 @@ def update(self) -> None:
             # update the cameras
             for cam_prefix, sensor_names in self._cameras.items():
                 for sensor_name in sensor_names:
-                    sens_obj = self._sim._sensors[sensor_name]._sensor_object
+                    sens_obj = self._sim.get_sensor(sensor_name).sensor_object
                     cam_info = self.params.cameras[cam_prefix]
 
                     if cam_info.attached_link_id == -1:
diff --git a/habitat-lab/habitat/sims/habitat_simulator/debug_visualizer.py b/habitat-lab/habitat/sims/habitat_simulator/debug_visualizer.py
@@ -341,11 +341,12 @@ def create_dbv_agent(
         )
         self.agent = self.sim.agents[-1]
         self.agent_id = len(self.sim.agents) - 1
-        self.sim._Simulator__sensors.append({})
-        self.sim._update_simulator_sensors(self.sensor_uuid, self.agent_id)
-        self.sensor = self.sim._Simulator__sensors[self.agent_id][
-            self.sensor_uuid
-        ]
+        # Wrap the C++ sensor in a Python Sensor and register it
+        cpp_sensor = self.agent.scene_node.subtree_sensors[self.sensor_uuid]
+        from habitat_sim.sensors.sensor_wrapper import Sensor as SimSensor
+
+        self.sensor = SimSensor(sim=self.sim, sensor_object=cpp_sensor)
+        self.sim.sensors[self.sensor_uuid] = self.sensor
 
     def remove_dbv_agent(self) -> None:
         """
@@ -358,9 +359,11 @@ def remove_dbv_agent(self) -> None:
 
         # NOTE: this guards against cases where the Simulator is deconstructed before the DBV
         if self.agent_id < len(self.sim.agents):
-            # remove the agent and sensor from the Simulator instance
+            # remove the sensor from the registry and agent from the Simulator
+            if self.sensor_uuid in self.sim.sensors:
+                self.sensor.close()
+                del self.sim.sensors[self.sensor_uuid]
             self.agent.close()
-            del self.sim._Simulator__sensors[self.agent_id]
             del self.sim.agents[self.agent_id]
 
         self.agent = None
diff --git a/habitat-lab/habitat/sims/habitat_simulator/habitat_simulator.py b/habitat-lab/habitat/sims/habitat_simulator/habitat_simulator.py
@@ -732,8 +732,8 @@ def add_keyframe_to_observations(self, observations):
         assert self.config.enable_batch_renderer
 
         assert KEYFRAME_OBSERVATION_KEY not in observations
-        for _sensor_uuid, sensor in self._sensors.items():
-            node = sensor._sensor_object.node
+        for _sensor_uuid, sensor in self.sensors.items():
+            node = sensor.node
             transform = node.absolute_transformation()
             rotation = mn.Quaternion.from_matrix(transform.rotation())
             self.gfx_replay_manager.add_user_transform_to_keyframe(
diff --git a/habitat-lab/habitat/tasks/nav/instance_image_nav_task.py b/habitat-lab/habitat/tasks/nav/instance_image_nav_task.py
@@ -22,7 +22,6 @@
 from habitat.tasks.nav.nav import NavigationEpisode
 from habitat.tasks.nav.object_nav_task import ObjectGoal, ObjectNavigationTask
 from habitat.utils.geometry_utils import quaternion_from_coeff
-from habitat_sim import bindings as hsim
 from habitat_sim.agent.agent import AgentState, SixDOFPose
 
 try:
@@ -168,15 +167,7 @@ def _add_sensor(
         )
 
     def _remove_sensor(self, sensor_uuid: str) -> None:
-        agent = self._sim.get_agent(0)
-        del self._sim._sensors[sensor_uuid]
-        hsim.SensorFactory.delete_subtree_sensor(agent.scene_node, sensor_uuid)
-        del agent._sensors[sensor_uuid]
-        agent.agent_config.sensor_specifications = [
-            s
-            for s in agent.agent_config.sensor_specifications
-            if s.uuid != sensor_uuid
-        ]
+        self._sim.remove_sensor(sensor_uuid)
 
     def _get_instance_image_goal(
         self, img_params: InstanceImageParameters
@@ -188,8 +179,9 @@ def _get_instance_image_goal(
         sensor_uuid = f"{self.cls_uuid}_sensor"
         self._add_sensor(img_params, sensor_uuid)
 
-        self._sim._sensors[sensor_uuid].draw_observation()
-        img = self._sim._sensors[sensor_uuid].get_observation()[:, :, :3]
+        sensor = self._sim.get_sensor(sensor_uuid)
+        sensor.draw_observation()
+        img = sensor.get_observation()[:, :, :3]
 
         self._remove_sensor(sensor_uuid)
         return img
diff --git a/habitat-lab/habitat/tasks/nav/nav.py b/habitat-lab/habitat/tasks/nav/nav.py
@@ -1048,9 +1048,9 @@ def _move_camera_vertical(self, amount: float):
         assert (
             len(self._sim.agents) == 1  # type: ignore
         ), "For navigation tasks, there can be only one agent in the scene"
-        sensor_names = list(self._sim.agents[0]._sensors.keys())  # type: ignore
+        sensor_names = list(self._sim.agents[0].sensors.keys())  # type: ignore
         for sensor_name in sensor_names:
-            sensor = self._sim.agents[0]._sensors[sensor_name].node  # type: ignore
+            sensor = self._sim.agents[0].sensors[sensor_name].node  # type: ignore
             sensor.rotation = sensor.rotation * mn.Quaternion.rotation(
                 mn.Deg(amount), mn.Vector3.x_axis()
             )
diff --git a/habitat-lab/habitat/tasks/rearrange/articulated_agent_manager.py b/habitat-lab/habitat/tasks/rearrange/articulated_agent_manager.py
@@ -104,7 +104,7 @@ def __init__(self, cfg: "SimulatorConfig", sim: "Simulator"):
                     for k, v in agent.params.cameras.items()
                 }
                 for camera_prefix in agent.params.cameras:
-                    for sensor_name in self._sim._sensors:
+                    for sensor_name in self._sim.sensors:
                         if sensor_name.startswith(camera_prefix):
                             agent._cameras[camera_prefix].append(sensor_name)
 
diff --git a/mypy.ini b/mypy.ini
@@ -1,5 +1,5 @@
 [mypy]
-disable_error_code=override
+disable_error_code = override
 
 # do not follow imports (except for ones found in typeshed)
 ignore_missing_imports = True
@@ -13,15 +13,14 @@ strict_optional = False
 warn_unused_configs = True
 warn_redundant_casts = True
 # ensure all execution paths are returning
-warn_no_return= True
+warn_no_return = True
 warn_unreachable = True
 allow_redefinition = True
 
 show_error_codes = True
 check_untyped_defs = True
 
-
-files=
+files =
     habitat-lab/habitat,
     habitat-baselines/habitat_baselines,
     test
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,5 +1,4 @@
 [tool.black]
-line_length = 79
 exclude = '''
 (
   /(
@@ -20,3 +19,4 @@ exclude = '''
   )/
 )
 '''
+line_length = 79
diff --git a/setup.cfg b/setup.cfg
@@ -1,16 +1,16 @@
 [aliases]
-test=pytest
+test = pytest
 
 [flake8]
 select = A,B,C,F,R,W,SIM
 exclude =
-	.git,
-	__pycache__,
-	build,
-	data,
-	dist,
-	docs,
-	src/deps
+    .git,
+    __pycache__,
+    build,
+    data,
+    dist,
+    docs,
+    src/deps
 max-line-length = 88
 # A003 prevents class attrs from having builtin name properties
 # C401, and C402 are ignored to make scanning between dict and set easy
@@ -21,14 +21,14 @@ max-line-length = 88
 # SIM106 has too many false positives
 # SIM113 has too many false positives
 ignore =
-	A003,
-	C401,C402,C408,
-	SIM105,SIM106,SIM113,
-	R504,
-	W503,
+    A003,
+    C401,C402,C408,
+    SIM105,SIM106,SIM113,
+    R504,
+    W503,
 per-file-ignores =
-	*/__init__.py:F401
-	examples/tutorials/nb_python/*.py:B008,F841
+    */__init__.py:F401
+    examples/tutorials/nb_python/*.py:B008,F841
 
 [tool:pytest]
 addopts = --verbose -rsxX -q
diff --git a/test/test_debug_visualizer.py b/test/test_debug_visualizer.py
@@ -35,21 +35,21 @@ def test_debug_visualizer():
     hab_cfg = make_cfg(sim_settings)
     with habitat_sim.Simulator(hab_cfg) as sim:
         # at first sim should have only the initial default rgb sensor and agent
-        assert len(sim._Simulator__sensors) == 1
+        assert len(sim.sensors) == 1
         assert len(sim.agents) == 1
 
         # initialize the dbv
         dbv = DebugVisualizer(sim)
 
         # before initializing nothing changes
-        assert len(sim._Simulator__sensors) == 1
+        assert len(sim.sensors) == 1
         assert len(sim.agents) == 1
 
         # create and register the agent/sensor
         dbv.create_dbv_agent()
 
         # now we should have two sensors and agents
-        assert len(sim._Simulator__sensors) == 2
+        assert len(sim.sensors) == 2
         assert len(sim.agents) == 2
 
         # collect all the debug visualizer observations for showing later
@@ -60,12 +60,12 @@ def test_debug_visualizer():
 
         # test removing the agent/sensor
         dbv.remove_dbv_agent()
-        assert len(sim._Simulator__sensors) == 1
+        assert len(sim.sensors) == 1
         assert len(sim.agents) == 1
 
         # test switching modes
         dbv.create_dbv_agent()
-        assert len(sim._Simulator__sensors) == 2
+        assert len(sim.sensors) == 2
         assert len(sim.agents) == 2
         assert dbv.agent is not None
         assert dbv.equirect == False
