Skip to content

Commit 3efc5be

Browse files
vmoensvmoens
committed
amend
1 parent 474fa6c commit 3efc5be

File tree

4 files changed

+126
-13
lines changed

4 files changed

+126
-13
lines changed

‎.github/unittest/linux_libs/scripts_habitat/download_datasets.sh

Lines changed: 95 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,21 +13,106 @@ mkdir -p data/datasets
1313
# Set environment variables for Habitat data paths
1414
export HABITAT_DATA_PATH="$(pwd)/data"
1515

16-
echo "Downloading Habitat test scenes..."
17-
python -m habitat_sim.utils.datasets_download --uids habitat_test_scenes --data-path data/
16+
# Note: Using manual downloads to avoid git-lfs prune issues with Habitat utility
1817

19-
echo "Downloading ReplicaCAD scenes..."
20-
python -m habitat_sim.utils.datasets_download --uids replica_cad --data-path data/
18+
# Function to download datasets manually (avoiding Habitat utility git-lfs issues)
19+
download_habitat_dataset() {
20+
local uid=$1
21+
local description=$2
22+
23+
echo "Downloading $description manually..."
24+
25+
case "$uid" in
26+
"habitat_test_scenes")
27+
# Manual download for test scenes
28+
cd data/scene_datasets
29+
if [ ! -d "habitat_test_scenes" ]; then
30+
git clone https://github.com/facebookresearch/habitat-test-scenes.git habitat_test_scenes || {
31+
echo "Manual download failed for $description"
32+
return 1
33+
}
34+
else
35+
echo "habitat_test_scenes already exists, skipping download"
36+
fi
37+
cd ../..
38+
;;
39+
"replica_cad")
40+
# Manual download for ReplicaCAD
41+
cd data/scene_datasets
42+
if [ ! -d "replica_cad" ]; then
43+
git clone https://github.com/facebookresearch/replica-cad.git replica_cad || {
44+
echo "Manual download failed for $description"
45+
return 1
46+
}
47+
else
48+
echo "replica_cad already exists, skipping download"
49+
fi
50+
cd ../..
51+
;;
52+
"habitat_test_pointnav_dataset")
53+
# Manual download for pointnav dataset
54+
cd data/datasets
55+
if [ ! -d "habitat_test_pointnav_dataset" ]; then
56+
wget -O habitat_test_pointnav_dataset.zip https://dl.fbaipublicfiles.com/habitat/data/datasets/pointnav/habitat-test-scenes/v1/habitat-test-scenes-v1.zip || {
57+
echo "Manual download failed for $description"
58+
return 1
59+
}
60+
unzip -o habitat_test_pointnav_dataset.zip
61+
rm habitat_test_pointnav_dataset.zip
62+
else
63+
echo "habitat_test_pointnav_dataset already exists, skipping download"
64+
fi
65+
cd ../..
66+
;;
67+
*)
68+
echo "Unknown dataset UID: $uid"
69+
return 1
70+
;;
71+
esac
72+
echo "$description downloaded successfully!"
73+
}
74+
75+
# Download datasets with fallback
76+
download_habitat_dataset "habitat_test_scenes" "Habitat test scenes"
77+
download_habitat_dataset "replica_cad" "ReplicaCAD scenes"
2178

2279
echo "Downloading rearrange pick dataset..."
2380
cd data/datasets
24-
wget -O rearrange_pick_replica_cad_v0.zip https://dl.fbaipublicfiles.com/habitat/data/datasets/rearrange_pick/replica_cad/v0/rearrange_pick_replica_cad_v0.zip
25-
unzip -o rearrange_pick_replica_cad_v0.zip
26-
rm rearrange_pick_replica_cad_v0.zip
81+
if [ ! -d "rearrange_pick_replica_cad_v0" ]; then
82+
wget -O rearrange_pick_replica_cad_v0.zip https://dl.fbaipublicfiles.com/habitat/data/datasets/rearrange_pick/replica_cad/v0/rearrange_pick_replica_cad_v0.zip
83+
unzip -o rearrange_pick_replica_cad_v0.zip
84+
rm rearrange_pick_replica_cad_v0.zip
85+
else
86+
echo "rearrange_pick_replica_cad_v0 already exists, skipping download"
87+
fi
2788
cd ../..
2889

29-
echo "Downloading point-goal navigation episodes for test scenes..."
30-
python -m habitat_sim.utils.datasets_download --uids habitat_test_pointnav_dataset --data-path data/
90+
download_habitat_dataset "habitat_test_pointnav_dataset" "Point-goal navigation episodes for test scenes"
3191

3292
echo "Datasets downloaded successfully!"
33-
ls -la data/
93+
94+
# Final verification
95+
echo "Verifying downloaded datasets..."
96+
echo "Scene datasets:"
97+
ls -la data/scene_datasets/ 2>/dev/null || echo "No scene_datasets directory found"
98+
echo "Task datasets:"
99+
ls -la data/datasets/ 2>/dev/null || echo "No datasets directory found"
100+
101+
# Check for required datasets
102+
required_scenes=0
103+
if [ -d "data/scene_datasets/habitat_test_scenes" ] || [ -d "data/scene_datasets/replica_cad" ]; then
104+
required_scenes=1
105+
fi
106+
107+
if [ -d "data/datasets/rearrange_pick_replica_cad_v0" ]; then
108+
required_datasets=1
109+
else
110+
required_datasets=0
111+
fi
112+
113+
if [ $required_scenes -eq 1 ] && [ $required_datasets -eq 1 ]; then
114+
echo "All required datasets are present!"
115+
else
116+
echo "ERROR: Some required datasets are missing!"
117+
exit 1
118+
fi

‎.github/unittest/linux_libs/scripts_habitat/run_all.sh

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,17 @@ bash ${this_dir}/setup_env.sh
2323
bash ${this_dir}/install.sh
2424

2525
# Download required Habitat datasets
26-
bash ${this_dir}/download_datasets.sh
26+
echo "Starting Habitat dataset download..."
27+
if bash ${this_dir}/download_datasets.sh; then
28+
echo "Habitat dataset download completed successfully!"
29+
else
30+
echo "ERROR: Habitat dataset download failed!"
31+
echo "Checking what was downloaded:"
32+
ls -la data/ 2>/dev/null || echo "No data directory found"
33+
ls -la data/scene_datasets/ 2>/dev/null || echo "No scene_datasets directory found"
34+
ls -la data/datasets/ 2>/dev/null || echo "No datasets directory found"
35+
exit 1
36+
fi
2737

2838
#apt-get install -y freeglut3 freeglut3-dev
2939
bash ${this_dir}/run_test.sh

‎.github/unittest/linux_libs/scripts_habitat/run_test.sh

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,24 @@ export MKL_THREADING_LAYER=GNU
3636

3737
# Set Habitat data path
3838
export HABITAT_DATA_PATH="$(pwd)/data"
39+
40+
# Check if required datasets are present
41+
echo "Checking for required Habitat datasets..."
42+
if [ ! -d "data/scene_datasets/habitat_test_scenes" ] && [ ! -d "data/scene_datasets/replica_cad" ]; then
43+
echo "ERROR: Required scene datasets not found!"
44+
echo "Available directories in data/scene_datasets:"
45+
ls -la data/scene_datasets/ 2>/dev/null || echo "No scene_datasets directory found"
46+
exit 1
47+
fi
48+
49+
if [ ! -d "data/datasets/rearrange_pick_replica_cad_v0" ]; then
50+
echo "ERROR: Rearrange pick dataset not found!"
51+
echo "Available directories in data/datasets:"
52+
ls -la data/datasets/ 2>/dev/null || echo "No datasets directory found"
53+
exit 1
54+
fi
55+
56+
echo "Required datasets found successfully!"
3957
# more logging
4058

4159
#wget https://github.com/openai/mujoco-py/blob/master/vendor/10_nvidia.json

‎.github/unittest/linux_libs/scripts_habitat/setup_env.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,9 @@ conda install -c anaconda cython="<3.0.0" -y
5252

5353
# 3. Install git LFS
5454
mkdir git_lfs
55-
wget https://github.com/git-lfs/git-lfs/releases/download/v2.9.0/git-lfs-linux-amd64-v2.9.0.tar.gz --directory-prefix git_lfs
55+
wget https://github.com/git-lfs/git-lfs/releases/download/v3.4.1/git-lfs-linux-amd64-v3.4.1.tar.gz --directory-prefix git_lfs
5656
cd git_lfs
57-
tar -xf git-lfs-linux-amd64-v2.9.0.tar.gz
57+
tar -xf git-lfs-linux-amd64-v3.4.1.tar.gz
5858
chmod 755 install.sh
5959
./install.sh
6060
cd ..

0 commit comments

Comments
 (0)