Skip to content

Commit 68acabd

Browse files
cgokmencgokmen
authored
Merge pull request #1980 from StanfordVL/profiling
Fix profiling
2 parents 25d30d5 + d0dce76 commit 68acabd

File tree

4 files changed

+252
-36
lines changed

4 files changed

+252
-36
lines changed

‎.github/workflows/profiling.yml‎

Lines changed: 243 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,261 @@
11
name: Profiling
22

3-
on:
4-
workflow_dispatch:
3+
on:
4+
pull_request:
55
push:
66
branches:
7-
- og-develop
7+
- main
8+
workflow_dispatch:
89

910
permissions:
10-
# deployments permission to deploy GitHub pages website
11-
deployments: write
12-
# contents permission to update profiling contents in gh-pages branch
13-
contents: write
11+
pull-requests: write
12+
contents: read
1413

1514
concurrency:
1615
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
1716
cancel-in-progress: true
1817

1918
jobs:
20-
profiling:
21-
name: Speed Profiling
19+
profile:
20+
name: "Profile: ${{ matrix.label }}"
2221
runs-on: [self-hosted, linux, gpu, dataset-enabled]
2322

24-
steps:
25-
- name: Fix home
26-
run: echo "HOME=/root" >> $GITHUB_ENV
23+
strategy:
24+
fail-fast: false
25+
matrix:
26+
include:
27+
# Baselines (GPU dynamics, flatcache)
28+
- name: baseline-empty
29+
args: "-g -f"
30+
label: "Empty scene"
31+
- name: baseline-empty-1robot
32+
args: "-g -f -r 1"
33+
label: "Empty scene + 1 Robot"
34+
- name: baseline-rs-int
35+
args: "-g -f -s Rs_int"
36+
label: "Rs_int"
37+
- name: baseline-rs-int-1robot
38+
args: "-g -f -s Rs_int -r 1"
39+
label: "Rs_int + 1 Robot"
40+
# Scenes (flatcache vs non-flatcache)
41+
- name: scene-house
42+
args: "-f -r 1 -s house_single_floor"
43+
label: "house_single_floor + 1 Robot"
44+
- name: scene-house-no-flatcache
45+
args: "-r 1 -s house_single_floor"
46+
label: "house_single_floor + 1 Robot (no flatcache)"
47+
# Non-physics features (GPU dynamics, flatcache)
48+
- name: feature-fluids
49+
args: "-g -f -r 1 -w"
50+
label: "1 Robot + fluids"
51+
- name: feature-cloth
52+
args: "-g -r 1 -c"
53+
label: "1 Robot + cloth"
54+
- name: feature-particles
55+
args: "-g -f -r 1 -p"
56+
label: "1 Robot + macro particles"
2757

58+
defaults:
59+
run:
60+
working-directory: ${{ github.workspace }}/OmniGibson
61+
62+
steps:
2863
- name: Checkout source
2964
uses: actions/checkout@v4
65+
with:
66+
submodules: true
67+
68+
- name: Install BDDL
69+
run: pip install -e .
70+
working-directory: ${{ github.workspace }}/bddl3
71+
72+
- name: Install OmniGibson
73+
run: pip install -e .[dev,primitives,eval] --no-build-isolation
74+
75+
- name: Run profiling
76+
run: |
77+
rm -f output.json
78+
python tests/benchmark/profiling.py ${{ matrix.args }}
79+
80+
- name: Upload results
81+
uses: actions/upload-artifact@v4
82+
if: always()
83+
with:
84+
name: profiling-${{ matrix.name }}
85+
path: OmniGibson/output.json
86+
if-no-files-found: warn
87+
88+
report:
89+
name: Compile Profiling Report
90+
runs-on: ubuntu-latest
91+
needs: [profile]
92+
if: always() && github.event_name == 'pull_request'
93+
permissions:
94+
pull-requests: write
95+
steps:
96+
- name: Download PR profiling results
97+
uses: actions/download-artifact@v4
98+
continue-on-error: true
99+
with:
100+
path: pr-results
101+
pattern: profiling-*
102+
103+
- name: Restore baseline from main
104+
uses: actions/cache/restore@v4
105+
continue-on-error: true
106+
with:
107+
path: baseline
108+
key: profiling-baseline-${{ github.event.pull_request.base.sha }}
109+
restore-keys: |
110+
profiling-baseline-
111+
112+
- name: Generate report
113+
run: |
114+
python3 << 'PYEOF'
115+
import json, os
116+
117+
SCENARIOS = [
118+
("profiling-baseline-empty", "Empty scene", "Baselines (GPU dynamics, flatcache)"),
119+
("profiling-baseline-empty-1robot", "Empty scene + 1 Robot", "Baselines (GPU dynamics, flatcache)"),
120+
("profiling-baseline-rs-int", "Rs_int", "Baselines (GPU dynamics, flatcache)"),
121+
("profiling-baseline-rs-int-1robot", "Rs_int + 1 Robot", "Baselines (GPU dynamics, flatcache)"),
122+
("profiling-scene-house", "house_single_floor + 1 Robot", "Scenes"),
123+
("profiling-scene-house-no-flatcache", "house_single_floor + 1 Robot (no FC)", "Scenes"),
124+
("profiling-feature-fluids", "1 Robot + fluids", "Non-physics features (GPU dynamics, flatcache)"),
125+
("profiling-feature-cloth", "1 Robot + cloth", "Non-physics features (GPU dynamics, flatcache)"),
126+
("profiling-feature-particles", "1 Robot + macro particles", "Non-physics features (GPU dynamics, flatcache)"),
127+
]
128+
129+
METRICS = ["FPS", "Loading time", "Isaac step time", "Non-Isaac step time", "Memory usage", "Vram usage"]
130+
HEADERS = {
131+
"FPS": "FPS",
132+
"Loading time": "Load (s)",
133+
"Isaac step time": "Isaac (ms)",
134+
"Non-Isaac step time": "Non-Isaac (ms)",
135+
"Memory usage": "Mem (GB)",
136+
"Vram usage": "VRAM (GB)",
137+
}
138+
HIGHER_IS_BETTER = {"FPS"}
30139
31-
- name: Install
32-
run: pip install -e .[dev,primitives]
33-
34-
- name: Run performance benchmark
35-
run: bash scripts/profiling.sh
36-
37-
- name: Store benchmark result
38-
uses: benchmark-action/github-action-benchmark@v1
39-
with:
40-
tool: 'customSmallerIsBetter'
41-
output-file-path: output.json
42-
benchmark-data-dir-path: profiling
43-
fail-on-alert: false
44-
alert-threshold: '200%'
45-
github-token: ${{ secrets.GITHUB_TOKEN }}
46-
comment-on-alert: false
47-
auto-push: true
140+
def load_results(base_path):
141+
results = {}
142+
if not os.path.isdir(base_path):
143+
return results
144+
for dir_name, _, _ in SCENARIOS:
145+
json_path = os.path.join(base_path, dir_name, "output.json")
146+
if not os.path.exists(json_path):
147+
continue
148+
with open(json_path) as f:
149+
data = json.load(f)
150+
metrics = {}
151+
for entry in data:
152+
metrics[entry["extra"][0]] = entry["value"]
153+
results[dir_name] = metrics
154+
return results
155+
156+
def fmt(metric, value):
157+
if value is None:
158+
return "N/A"
159+
return f"{value:.1f}" if metric in ("FPS", "Loading time") else f"{value:.2f}"
160+
161+
def delta(metric, pr_val, base_val):
162+
if pr_val is None or base_val is None or base_val == 0:
163+
return ""
164+
pct = ((pr_val - base_val) / abs(base_val)) * 100
165+
is_better = (pct > 0) if metric in HIGHER_IS_BETTER else (pct < 0)
166+
if abs(pct) < 5:
167+
emoji = "⚪"
168+
elif is_better:
169+
emoji = "🟢"
170+
else:
171+
emoji = "🔴"
172+
sign = "+" if pct > 0 else ""
173+
return f" ({sign}{pct:.1f}% {emoji})"
174+
175+
pr = load_results("pr-results")
176+
base = load_results("baseline")
177+
has_base = len(base) > 0
178+
179+
lines = ["## Profiling Report\n"]
180+
if not has_base:
181+
lines.append("> **Note:** No baseline found from `main`. Showing absolute values only.\n")
182+
183+
current_group = None
184+
for dir_name, label, group in SCENARIOS:
185+
if group != current_group:
186+
current_group = group
187+
lines.append(f"\n### {group}\n")
188+
hdr = "| Scenario |" + "".join(f" {HEADERS[m]} |" for m in METRICS)
189+
sep = "|---|" + "---|" * len(METRICS)
190+
lines.append(hdr)
191+
lines.append(sep)
192+
193+
pr_m = pr.get(dir_name, {})
194+
if not pr_m:
195+
lines.append(f"| {label} |" + " N/A |" * len(METRICS))
196+
continue
197+
198+
base_m = base.get(dir_name, {})
199+
row = f"| {label}"
200+
for m in METRICS:
201+
pv, bv = pr_m.get(m), base_m.get(m)
202+
cell = fmt(m, pv)
203+
if has_base and pv is not None:
204+
cell += delta(m, pv, bv)
205+
row += f" | {cell}"
206+
lines.append(row + " |")
207+
208+
lines.append("")
209+
lines.append("<details><summary>Legend</summary>\n")
210+
lines.append("- 🟢 Improvement >= 5%")
211+
lines.append("- 🔴 Regression >= 5%")
212+
lines.append("- ⚪ Change < 5%")
213+
lines.append("- For **FPS**, higher is better. For all other metrics, lower is better.")
214+
lines.append("\n</details>")
215+
216+
report = "\n".join(lines)
217+
with open("profiling-report.md", "w") as f:
218+
f.write(report)
219+
print(report)
220+
PYEOF
221+
222+
- name: Post PR comment
223+
uses: marocchino/sticky-pull-request-comment@v2
224+
with:
225+
header: profiling-report
226+
path: profiling-report.md
227+
228+
save_baseline:
229+
name: Save Baseline
230+
runs-on: ubuntu-latest
231+
needs: [profile]
232+
if: always() && github.event_name == 'push' && github.ref == 'refs/heads/main'
233+
steps:
234+
- name: Restore previous baseline
235+
uses: actions/cache/restore@v4
236+
continue-on-error: true
237+
with:
238+
path: baseline
239+
key: profiling-baseline-do-not-match
240+
restore-keys: |
241+
profiling-baseline-
242+
243+
- name: Download new profiling results
244+
uses: actions/download-artifact@v4
245+
continue-on-error: true
246+
with:
247+
path: new-results
248+
pattern: profiling-*
249+
250+
- name: Merge into baseline
251+
run: |
252+
mkdir -p baseline
253+
if [ -d new-results ]; then
254+
cp -r new-results/* baseline/
255+
fi
256+
257+
- name: Save baseline cache
258+
uses: actions/cache/save@v4
259+
with:
260+
path: baseline
261+
key: profiling-baseline-${{ github.sha }}

‎OmniGibson/omnigibson/utils/profiling_utils.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ def step(self, action):
107107
ret = [total_frame_time, omni_time, og_time, memory_usage, vram_usage]
108108
if self._current_step % 100 == 0:
109109
print(
110-
"total time: {:.3f} ms, Omni time: {:.3f} ms, OG time: {:.3f} ms, memory: {:.3f} GB, vram: {:.3f} GB.".format(
110+
"total time: {:.3f} ms, Isaac time: {:.3f} ms, Non-Isaac time: {:.3f} ms, memory: {:.3f} GB, vram: {:.3f} GB.".format(
111111
*ret
112112
)
113113
)

‎OmniGibson/tests/benchmark/profiling.py‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,9 @@
2121
parser.add_argument("-w", "--fluids", action="store_true")
2222
parser.add_argument("-g", "--gpu_dynamics", action="store_true")
2323
parser.add_argument("-p", "--macro_particle_system", action="store_true")
24+
parser.add_argument("-f", "--flatcache", action="store_true")
2425

25-
PROFILING_FIELDS = ["FPS", "Omni step time", "Non-omni step time", "Memory usage", "Vram usage"]
26+
PROFILING_FIELDS = ["FPS", "Isaac step time", "Non-Isaac step time", "Memory usage", "Vram usage"]
2627
NUM_CLOTH = 5
2728
NUM_SLICE_OBJECT = 3
2829

@@ -39,10 +40,10 @@
3940
def main():
4041
args = parser.parse_args()
4142
# Modify macros settings
42-
gm.ENABLE_HQ_RENDERING = args.fluids
43+
# gm.ENABLE_HQ_RENDERING = args.fluids # Temporarily disabled, since it requires >= 60 FPS
4344
gm.ENABLE_OBJECT_STATES = True
4445
gm.ENABLE_TRANSITION_RULES = True
45-
gm.ENABLE_FLATCACHE = not args.cloth
46+
gm.ENABLE_FLATCACHE = args.flatcache
4647
gm.USE_GPU_DYNAMICS = args.gpu_dynamics
4748

4849
cfg = {
@@ -56,7 +57,7 @@ def main():
5657
for i in range(args.robot):
5758
cfg["robots"].append(
5859
{
59-
"type": "Fetch",
60+
"model": "r1pro",
6061
"obs_modalities": ["rgb"],
6162
"position": [-1.3 + 0.75 * i + SCENE_OFFSET[args.scene][0], 0.5 + SCENE_OFFSET[args.scene][1], 0],
6263
"orientation": [0.0, 0.0, 0.7071, -0.7071],
@@ -148,7 +149,7 @@ def main():
148149
)
149150
knife.keep_still()
150151
if args.fluids:
151-
table.states[Covered].set_value(env.scene.get_system("water"))
152+
table.states[Covered].set_value(env.scene.get_system("water"), True)
152153

153154
output, results = [], []
154155

‎setup.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/bin/bash
2-
set -e -x
2+
set -e
33

44
# Parse arguments
55
HELP=false
@@ -231,6 +231,7 @@ if [ "$NEW_ENV" = true ]; then
231231
pip install torch==2.6.0 torchvision==0.21.0 torchaudio==2.6.0 --index-url https://download.pytorch.org/whl/cu${CUDA_VER_SHORT}
232232
echo "✓ PyTorch installation completed"
233233
fi
234+
234235
# Install BDDL
235236
if [ "$BDDL" = true ]; then
236237
echo "Installing BDDL..."

0 commit comments

Comments
 (0)