[CI,BugFix] Fix Habitat CI by upgrading to Python 3.10 and building habitat-sim from source (#3346)

Author: vmoens

.github/unittest/linux_libs/scripts_habitat/run_all.sh

@@ -5,12 +5,18 @@ set -v
 
 
 apt-get update && apt-get upgrade -y
-apt-get install -y vim git wget cmake
+apt-get install -y vim git wget cmake ninja-build
 
-apt-get install -y libglfw3 libgl1-mesa-glx libosmesa6 libglew-dev
+# OpenGL/EGL dependencies for headless rendering
+apt-get install -y libglfw3 libglfw3-dev libgl1-mesa-glx libosmesa6 libosmesa6-dev libglew-dev
 apt-get install -y libglvnd0 libgl1 libglx0 libegl1 libgles2
+apt-get install -y libegl1-mesa-dev libgles2-mesa-dev
 
+# Build tools and libraries for habitat-sim
 apt-get install -y g++ gcc
+apt-get install -y libjpeg-dev libpng-dev
+apt-get install -y pkg-config
+
 #apt-get upgrade -y libstdc++6
 #apt-get install -y libgcc
 apt-get dist-upgrade -y

.github/unittest/linux_libs/scripts_habitat/run_test.sh

@@ -6,46 +6,33 @@ set -v
 eval "$(./conda/bin/conda shell.bash hook)"
 conda activate ./env
 
-# we can install this now but not before installing tensordict and torchrl, g++ version will break the compilation
-# https://stackoverflow.com/questions/72540359/glibcxx-3-4-30-not-found-for-librosa-in-conda-virtual-environment-after-tryin
-#conda install -y -c conda-forge gcc=12.1.0
-conda install -y -c conda-forge libstdcxx-ng=12
-conda env config vars set \
-  MAX_IDLE_COUNT=1000 \
-  LD_PRELOAD=$LD_PRELOAD:$STDC_LOC TOKENIZERS_PARALLELISM=true
-
-## find libstdc
-STDC_LOC=$(find conda/ -name "libstdc++.so.6" | head -1)
+# Note: We no longer install libstdcxx-ng=12 here since we're building habitat-sim from source
+# and the pip-installed PyTorch has its own compatible libstdc++
 
 export PYTORCH_TEST_WITH_SLOW='1'
 export LAZY_LEGACY_OP=False
-python -m torch.utils.collect_env
+
 # Avoid error: "fatal: unsafe repository"
 git config --global --add safe.directory '*'
 
 root_dir="$(git rev-parse --show-toplevel)"
 env_dir="${root_dir}/env"
 lib_dir="${env_dir}/lib"
 
-# smoke test
-python -c "import habitat;import habitat.gym"
-
-# solves ImportError: /lib64/libstdc++.so.6: version `GLIBCXX_3.4.21' not found
+# Set library path to include environment lib
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$lib_dir
 export MKL_THREADING_LAYER=GNU
-# more logging
 
-#wget https://github.com/openai/mujoco-py/blob/master/vendor/10_nvidia.json
-#mv 10_nvidia.json /usr/share/glvnd/egl_vendor.d/10_nvidia.json
+# Verify torch works
+python -m torch.utils.collect_env
 
+# Set habitat environment variables
 conda env config vars set \
   MAX_IDLE_COUNT=1000 \
   MAGNUM_LOG=quiet HABITAT_SIM_LOG=quiet TOKENIZERS_PARALLELISM=true
-
 conda deactivate && conda activate ./env
 
-
-# this workflow only tests the libs
+# smoke test
 python -c "import habitat;import habitat.gym"
 python -c """from torchrl.envs.libs.habitat import HabitatEnv
 env = HabitatEnv('HabitatRenderPick-v0')

.github/unittest/linux_libs/scripts_habitat/setup_env.sh

@@ -49,17 +49,26 @@ conda deactivate && conda activate "${env_dir}"
 pip3 install "cython<3"
 conda install -c anaconda cython="<3.0.0" -y
 
-
-# 3. Install git LFS
-mkdir git_lfs
-wget https://github.com/git-lfs/git-lfs/releases/download/v2.9.0/git-lfs-linux-amd64-v2.9.0.tar.gz --directory-prefix git_lfs
-cd git_lfs
-tar -xf git-lfs-linux-amd64-v2.9.0.tar.gz
-chmod 755 install.sh
-./install.sh
-cd ..
+# 3. Install git LFS (newer version that supports git lfs prune -f)
+mkdir -p git_lfs_tmp
+cd git_lfs_tmp
+wget https://github.com/git-lfs/git-lfs/releases/download/v3.4.0/git-lfs-linux-amd64-v3.4.0.tar.gz
+tar -xf git-lfs-linux-amd64-v3.4.0.tar.gz
+# The binary is in git-lfs-3.4.0/git-lfs
+chmod 755 git-lfs-3.4.0/git-lfs
+# Install to /usr/local/bin so it's available system-wide
+cp git-lfs-3.4.0/git-lfs /usr/local/bin/
+cd "${root_dir}"
 git lfs install
 
+# Configure git-lfs for better performance (higher timeouts, more concurrent transfers)
+git config --global lfs.activitytimeout 600
+git config --global lfs.dialtimeout 60
+git config --global lfs.tlstimeout 60
+git config --global lfs.concurrenttransfers 8
+git config --global http.version HTTP/1.1
+rm -rf git_lfs_tmp
+
 # 4. Install Conda dependencies
 printf "* Installing dependencies (except PyTorch)\n"
 echo "  - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml"
@@ -69,7 +78,37 @@ pip install pip --upgrade
 
 conda env update --file "${this_dir}/environment.yml" --prune
 
-conda install habitat-sim withbullet headless -c conda-forge -c aihabitat -y
+# 5. Install habitat-sim from source (conda packages don't support Python 3.10+)
+# Install build dependencies
+pip3 install ninja numpy
+
+# Clone and build habitat-sim from source
+cd "${root_dir}"
+git clone --branch stable https://github.com/facebookresearch/habitat-sim.git --recursive
+cd habitat-sim
+
+# Build with headless (EGL) and bullet physics support
+# Ensure system cmake is used (pip cmake 4.x is incompatible with habitat-sim's CMake files)
+# Put /usr/bin at the front of PATH to prefer system cmake over any pip-installed cmake
+export PATH="/usr/bin:$PATH"
+# Also set CMAKE_EXECUTABLE to explicitly use system cmake
+export CMAKE_EXECUTABLE=/usr/bin/cmake
+pip3 install . --no-build-isolation
+
+cd "${root_dir}"
+
+# 6. Download required Habitat test datasets
+# Use datasets_download with git-lfs configured for better performance
+echo "$(date): Starting dataset downloads..."
+
+# Download all rearrange task assets (includes replica_cad, ycb, robots, and episode data)
+python -m habitat_sim.utils.datasets_download --uids rearrange_task_assets --data-path data/ --no-prune
+
+echo "$(date): Dataset downloads complete!"
+echo "Total data size:"
+du -sh data/
+
+# Install habitat-lab
 git clone https://github.com/facebookresearch/habitat-lab.git
 cd habitat-lab
 pip3 install -e habitat-lab

.github/workflows/test-linux-habitat.yml

@@ -23,7 +23,7 @@ jobs:
   tests:
     strategy:
       matrix:
-        python_version: ["3.9"]
+        python_version: ["3.10"]
         cuda_arch_version: ["12.8"]
       fail-fast: false
     uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main