Skip to content

Commit de55d1b

Browse files
kew6688wangyukai
kew6688
authored and
wangyukai
committed
[feat] add challenge page; add task metric
1 parent 09df28f commit de55d1b

20 files changed

Lines changed: 1605 additions & 841 deletions

File tree

‎astro.config.mjs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,11 @@ export default defineConfig({
135135
},
136136
],
137137
},
138+
{
139+
slug: 'challenge',
140+
label: 'Challenge',
141+
translations: { 'zh-CN': '挑战赛', fr: 'Challenge', ja: 'Challenge', ko: 'Challenge', de: 'Challenge', es: 'Challenge' },
142+
},
138143
],
139144
}),
140145
],

‎src/components/TaskShowcase.astro‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
---
2+
import { getLocalizedText, showcaseCategories, showcaseCopy, type ShowcaseLocale } from '../data/taskShowcase';
3+
4+
interface Props {
5+
locale?: ShowcaseLocale;
6+
}
7+
8+
const { locale = 'en' } = Astro.props;
9+
const copy = showcaseCopy[locale] ?? showcaseCopy.en;
10+
---
11+
12+
<div class="task-showcase not-content">
13+
<p class="task-showcase-intro">{copy.intro}</p>
14+
15+
{
16+
showcaseCategories.map((category) => {
17+
const sectionId = `showcase-${category.id}`;
18+
return (
19+
<section class="task-showcase-section" data-showcase-section>
20+
<h2 id={sectionId}>{getLocalizedText(category.title, locale)}</h2>
21+
<div class="showcase-grid not-content">
22+
{
23+
category.tasks.map((task) => (
24+
<div class="showcase-card">
25+
<video src={task.video} autoplay loop muted playsinline></video>
26+
<span class="showcase-label">{getLocalizedText(task.label, locale)}</span>
27+
</div>
28+
))
29+
}
30+
</div>
31+
<div class="task-tabs" data-task-tabs>
32+
<div class="task-tab-list" role="tablist" aria-labelledby={sectionId}>
33+
{
34+
category.tasks.map((task, index) => (
35+
<button
36+
type="button"
37+
class:list={['task-tab-button', { active: index === 0 }]}
38+
role="tab"
39+
aria-selected={index === 0 ? 'true' : 'false'}
40+
data-task-tab
41+
data-target={`${category.id}-${task.id}`}
42+
>
43+
{getLocalizedText(task.label, locale)}
44+
</button>
45+
))
46+
}
47+
</div>
48+
49+
<div class="task-panels">
50+
{
51+
category.tasks.map((task, index) => (
52+
<article
53+
class:list={['task-panel', { active: index === 0 }]}
54+
role="tabpanel"
55+
data-task-panel
56+
data-panel-id={`${category.id}-${task.id}`}
57+
>
58+
<div class="task-panel-media">
59+
<video src={task.video} autoplay loop muted playsinline controls></video>
60+
</div>
61+
<div class="task-panel-meta">
62+
<h3>{getLocalizedText(task.label, locale)}</h3>
63+
<dl class="task-meta-list">
64+
<div>
65+
<dt>{copy.location}</dt>
66+
<dd>{getLocalizedText(task.location, locale)}</dd>
67+
</div>
68+
<div>
69+
<dt>{copy.instruction}</dt>
70+
<dd>{getLocalizedText(task.instruction, locale)}</dd>
71+
</div>
72+
<div>
73+
<dt>{copy.score}</dt>
74+
<dd>{getLocalizedText(task.score, locale)}</dd>
75+
</div>
76+
</dl>
77+
</div>
78+
</article>
79+
))
80+
}
81+
</div>
82+
</div>
83+
</section>
84+
);
85+
})
86+
}
87+
</div>
88+
89+
<script>
90+
const setupTaskTabs = () => {
91+
document.querySelectorAll('[data-task-tabs]').forEach((root) => {
92+
const tabs = Array.from(root.querySelectorAll('[data-task-tab]'));
93+
const panels = Array.from(root.querySelectorAll('[data-task-panel]'));
94+
95+
tabs.forEach((tab) => {
96+
tab.addEventListener('click', () => {
97+
const target = tab.getAttribute('data-target');
98+
tabs.forEach((item) => {
99+
item.classList.toggle('active', item === tab);
100+
item.setAttribute('aria-selected', item === tab ? 'true' : 'false');
101+
});
102+
panels.forEach((panel) => {
103+
panel.classList.toggle('active', panel.getAttribute('data-panel-id') === target);
104+
});
105+
});
106+
});
107+
});
108+
};
109+
110+
if (document.readyState === 'loading') {
111+
document.addEventListener('DOMContentLoaded', setupTaskTabs, { once: true });
112+
} else {
113+
setupTaskTabs();
114+
}
115+
</script>

‎src/components/starlight/NavIcon.astro‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ const iconMarkup: Record<string, string> = {
1313
'<path d="M10 3.2v7.4" stroke="currentColor" stroke-width="1.6" stroke-linecap="round"/><path d="m6.8 8.9 3.2 3.2 3.2-3.2" stroke="currentColor" stroke-width="1.6" stroke-linecap="round" stroke-linejoin="round"/><path d="M4.2 14.6h11.6" stroke="currentColor" stroke-width="1.6" stroke-linecap="round"/>',
1414
grid:
1515
'<rect x="3.4" y="3.4" width="5.2" height="5.2" rx="1.2" stroke="currentColor" stroke-width="1.4"/><rect x="11.4" y="3.4" width="5.2" height="5.2" rx="1.2" stroke="currentColor" stroke-width="1.4"/><rect x="3.4" y="11.4" width="5.2" height="5.2" rx="1.2" stroke="currentColor" stroke-width="1.4"/><rect x="11.4" y="11.4" width="5.2" height="5.2" rx="1.2" stroke="currentColor" stroke-width="1.4"/>',
16+
leaderboard:
17+
'<path d="M4.4 15.8h11.2" stroke="currentColor" stroke-width="1.5" stroke-linecap="round"/><path d="M6 15.8V9.8" stroke="currentColor" stroke-width="1.8" stroke-linecap="round"/><path d="M10 15.8V5.6" stroke="currentColor" stroke-width="1.8" stroke-linecap="round"/><path d="M14 15.8v-7.9" stroke="currentColor" stroke-width="1.8" stroke-linecap="round"/><circle cx="10" cy="4.1" r="1.6" fill="currentColor"/><path d="M7.9 3.9 6.8 3M12.1 3.9 13.2 3" stroke="currentColor" stroke-width="1.3" stroke-linecap="round"/>',
1618
cloud:
1719
'<path d="M6.4 14.6h7.3a2.8 2.8 0 0 0 .2-5.6 4.2 4.2 0 0 0-8-1.2A3.2 3.2 0 0 0 6.4 14.6Z" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>',
1820
globe:

‎src/components/starlight/navIconName.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ export const getNavIconName = (label = '', href = '') => {
1515
if (text.includes('download') || text.includes('资产')) return 'download';
1616
if (text.includes('evaluation') || text.includes('评测') || text.includes('测试')) return 'grid';
1717
if (text.includes('run') || text.includes('运行')) return 'nodes';
18+
if (text.includes('challenge') || text.includes('leaderboard') || text.includes('榜单') || text.includes('挑战赛')) return 'leaderboard';
1819
if (text.includes('gmp') || text.includes('cli') || text.includes('code')) return 'code';
1920
if (text.includes('install') || text.includes('安装')) return 'file';
2021
if (text.includes('feature') || text.includes('功能')) return 'grid';

‎src/content/docs/challenge.md‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
---
2+
title: Challenge
3+
description: Submit your EBench results to the online challenge leaderboard.
4+
---
5+
6+
The EBench Challenge supports online submission for benchmark results. Follow the steps below to prepare a valid run and submit it to the leaderboard service.
7+
8+
## Baseline and Getting Started
9+
10+
Before submitting online, make sure you can run the benchmark locally:
11+
12+
- Set up the server and client environments in [Environment Setup](/getting-started/environment/).
13+
- Prepare the required benchmark assets from [Asset & Dataset](/getting-started/assets/).
14+
- Run a local benchmark first with [Run Evaluation](/evaluation/run-benchmark/).
15+
- If you use your own policy, follow [Integrate Your Own Model](/evaluation/custom-model/).
16+
17+
You should verify that your local run finishes normally and produces a complete result directory before attempting online submission.
18+
19+
## Online Submit Steps
20+
21+
The online workflow has three stages: create an online task, wait for the evaluation endpoint, and run evaluation workers against that endpoint.
22+
23+
### 1. Get your token
24+
25+
Open the platform landing page:
26+
27+
```text
28+
https://internrobotics-staging.shlab.org.cn/eval/landing-page
29+
```
30+
31+
Then:
32+
33+
1. Sign in to the platform.
34+
2. Open the API key or secret management page.
35+
3. Create a new API key and copy the token value.
36+
37+
### 2. Prepare the client environment
38+
39+
```bash
40+
git clone https://gitee.pjlab.org.cn/L2/MultimodalVLA/GenManip-Client.git
41+
cd GenManip-Client
42+
conda create -n client python=3.11 -y
43+
conda activate client
44+
pip install -e .
45+
```
46+
47+
### 3. Create an online evaluation task
48+
49+
Use `gmp online submit` to request a remote evaluation job:
50+
51+
```bash
52+
gmp online submit \
53+
--base_url https://internrobotics-staging.shlab.org.cn/eval \
54+
--token "$EBENCH_SUBMIT_TOKEN" \
55+
--benchmark_set EBench \
56+
--model_name internVLA \
57+
--model_type VLA
58+
```
59+
60+
After the backend task is ready, the command returns fields like:
61+
62+
```json
63+
{
64+
"task_id": "9ea5fb6ae980430da626958c4433ea18",
65+
"endpoint": "https://internrobotics-staging.shlab.org.cn/evalserver/9391d9e8/api/predict/embodied_eval.genmanip_eas_1_master"
66+
}
67+
```
68+
69+
Record both values:
70+
71+
- `task_id`: use this as the `run_id` when running evaluation.
72+
- `endpoint`: use this as the remote evaluation URL.
73+
74+
### 4. Start evaluation workers
75+
76+
Run the evaluator against the returned endpoint.
77+
78+
```bash
79+
gmp eval \
80+
--url "$EBENCH_ONLINE_ENDPOINT" \
81+
--token "$EBENCH_SUBMIT_TOKEN" \
82+
--run_id "$EBENCH_TASK_ID" \
83+
-a r5a \
84+
-g lift2 \
85+
-chunk_size 40 \
86+
--worker_id 0
87+
```
88+
89+
If you want to use the second backend worker, open another terminal and start:
90+
91+
```bash
92+
gmp eval \
93+
--url "$EBENCH_ONLINE_ENDPOINT" \
94+
--token "$EBENCH_SUBMIT_TOKEN" \
95+
--run_id "$EBENCH_TASK_ID" \
96+
-a r5a \
97+
-g lift2 \
98+
-chunk_size 40 \
99+
--worker_id 1
100+
```
101+
102+
### 5. Monitor the task
103+
104+
After the online task is created, the platform page will show the corresponding task. The final evaluation outputs are written to the same remote task record.
105+
106+
107+
## Online Submit URL
108+
109+
Create tasks through the official platform base URL:
110+
111+
```text
112+
https://internrobotics-staging.shlab.org.cn/eval
113+
```
114+
115+
After `gmp online submit`, use the returned per-task endpoint for evaluation:
116+
117+
```text
118+
https://internrobotics-staging.shlab.org.cn/evalserver/<task-endpoint>
119+
```
120+
121+
## Scoring Rules
122+
123+
- Each evaluated episode produces a task score between `0.0` and `1.0`.
124+
- A task receives full score when the required goal condition is completed within the episode; otherwise it receives `0.0`.
125+
- The leaderboard score is the average task score across the evaluated episodes in the submitted benchmark set.
126+
- For task-specific success semantics, see [Task Showcase](/evaluation/task-showcase/), where each task includes its `Location`, `Instruction`, and `Score` description.
127+
128+
## Example Checklist
129+
130+
- Baseline or custom model runs locally
131+
- Correct benchmark track and split selected
132+
- Submission token configured
133+
- Online submit URL confirmed
134+
- Result files ready for upload

0 commit comments

Comments
 (0)