[Fix] Habitat Refactor & Distributed VLNPE Refactor (#168)

* update vln yaml; fix import agent

* update habitat, using evaluator and config; env and agent is WIP

* add distributed_base evaluator

* Habitat env applied, distributed evaluator applied; clean evaluator and agent created

* fix observation issues

* update new register name; tiny fix on style

* latest tested

* delete temp agent; rename default evaluator for habitat

* update slurm bash

* update readme

* fix init dist print

* fix eval config; fix local rank to rank

* update init distributed mode if condition

* update dist for dlc

* fix bug in evaluator

* update distributed vln multi, episode loader

* cma tested

* cma tested; episode loader; torchrun; local agent tested; TODO fix isaac sim gpu setting

* add vlnpe distributed script

* add grscene; add new result write and resumble load data feature based on lmdb path + rank; remove TODOs and unused files

* fix bugs in evaluator and dataset for distributed; n1 and rdp tested

* fix comm log concurrency issue, create file with exist true

* fix progress log mkdir race condition

* fix comments

* polish existing configs and bash

* update bash align with doc

* fix CI test

* rename habitat_extensions

* fix comments

* remove useless line

* [file] Update 3D Printing Files for Camera of Unitree Go2 (#176)

* [file] Update 3d printing files for camera of Unitree Go2.

* [Doc] Update readme for 3d printing models.

* fix visualize image size; fix unused comment code; fixed evaluator name

* bump to version v0.0.2

* update vlnmulti to VLN; update habitatVln to HabitatVLN

* :Revert "Merge branch 'main' into vlnpe_refactor"

This reverts commit 7a23b72, reversing
changes made to ea73075.

---------

Co-authored-by: Yuqiang Yang <[email protected]>

Author: kew6688

internnav/agent/internvla_n1_agent.py

@@ -5,9 +5,9 @@
 
 class AgentCfg(BaseModel):
     server_host: str = 'localhost'
-    server_port: int = 5000
+    server_port: int = 8087
     model_name: str
-    ckpt_path: str
+    ckpt_path: str = None
     model_settings: Dict[str, Any]
 
 

internnav/configs/agent/__init__.py

@@ -59,9 +59,9 @@ class EvalCfg(BaseModel):
     eval_type: Optional[str] = None
     eval_settings: Optional[Dict[str, Any]] = {}
     agent: Optional[AgentCfg] = None
-    env: EnvCfg
-    task: TaskCfg
-    dataset: EvalDatasetCfg
+    env: EnvCfg = None
+    task: TaskCfg = None
+    dataset: EvalDatasetCfg = None
 
 
 __all__ = [

internnav/configs/evaluator/__init__.py

@@ -1,7 +1,13 @@
+import os
+import sys
 from typing import Any, Dict, List
 
 from internnav.configs.evaluator import EnvCfg, TaskCfg
 from internnav.env import base
+from internnav.env.utils.episode_loader import (
+    ResumablePathKeyEpisodeloader,
+    generate_vln_episode,
+)
 
 
 @base.Env.register('internutopia')
@@ -22,6 +28,23 @@ def __init__(self, env_config: EnvCfg, task_config: TaskCfg):
         super().__init__(env_config, task_config)
         env_settings = self.env_config.env_settings
         task_settings = self.task_config.task_settings
+
+        # generate episodes
+        self.episode_loader = ResumablePathKeyEpisodeloader(
+            env_settings['dataset'].dataset_type,
+            **env_settings['dataset'].dataset_settings,
+            rank=env_settings['rank'],
+            world_size=env_settings['world_size']
+        )
+        self.episodes = generate_vln_episode(self.episode_loader, task_config)
+        if len(self.episodes) == 0:
+            print("No episodes found for the given configuration.")
+            sys.exit(0)
+        task_settings.update({'episodes': self.episodes})
+
+        # set visible device for isaac sim
+        os.environ["CUDA_VISIBLE_DEVICES"] = str(env_settings.get('local_rank', 0))
+
         config = Config(
             simulator=SimConfig(**env_settings),
             env_num=task_settings['env_num'],

internnav/env/internutopia_env.py

@@ -0,0 +1,2 @@
+from .generate_episode import generate_vln_episode
+from .resumable import ResumablePathKeyEpisodeloader

internnav/env/utils/episode_loader/__init__.py

@@ -1,9 +1,7 @@
-from internnav.evaluator.utils.common import load_data
+from .dataset_utils import load_data, revise_one_data, skip_list
 
-from .data_reviser import revise_one_data, skip_list
 
-
-class BasePathKeyDataloader:
+class BasePathKeyEpisodeloader:
     def __init__(
         self,
         dataset_type,
@@ -13,7 +11,15 @@ def __init__(
         filter_same_trajectory,
         revise_data=True,
         filter_stairs=True,
+        rank=0,
+        world_size=1,
     ):
+        # current supported dataset types in InternUtopia
+        # only kujiale has special scene path
+        # others type should be considered the same as mp3d in loading
+        allowed = ('R2RVLN', 'mp3d', 'kujiale', 'grscene')
+        assert dataset_type in allowed, f"Unsupported dataset type: {dataset_type}. Allowed: {allowed}"
+
         self.path_key_data = {}
         self.path_key_scan = {}
         self.path_key_split = {}
@@ -25,14 +31,19 @@ def __init__(
                 filter_same_trajectory=filter_same_trajectory,
                 filter_stairs=filter_stairs,
                 dataset_type=dataset_type,
+                rank=rank,
+                world_size=world_size,
             )
             for scan, path_list in load_data_map.items():
                 for path in path_list:
                     trajectory_id = path['trajectory_id']
-                    if revise_data:
+
+                    # tiny revision for R2R dataset in MP3D to fit vlnpe task
+                    if dataset_type == 'mp3d' and revise_data:
                         if trajectory_id in skip_list:
                             continue
                         path = revise_one_data(path)
+
                     episode_id = path['episode_id']
                     path_key = f'{trajectory_id}_{episode_id}'
                     path['start_position'] += robot_offset