Skip to content

Commit 987118a

Browse files
k1000daiclaude
andcommitted
Preserve valid flags when merging datasets
Without this, merging validated datasets dropped the per-episode valid flag, so a later convert --valid-only would include episodes already known to be invalid. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LYnRMrL29hDAgjePm2DgZJ
1 parent 4d00a63 commit 987118a

2 files changed

Lines changed: 22 additions & 7 deletions

File tree

src/openarm_dataset/merge.py

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -131,13 +131,14 @@ def _build_merged_episodes(
131131

132132
for ds_idx, ds in enumerate(datasets):
133133
for ep in ds.meta.episodes:
134-
merged_episodes.append(
135-
{
136-
"id": str(len(merged_episodes)),
137-
"success": ep["success"],
138-
"task_index": task_index_maps[ds_idx][ep["task_index"]],
139-
}
140-
)
134+
merged_episode = {
135+
"id": str(len(merged_episodes)),
136+
"success": ep["success"],
137+
"task_index": task_index_maps[ds_idx][ep["task_index"]],
138+
}
139+
if "valid" in ep:
140+
merged_episode["valid"] = ep["valid"]
141+
merged_episodes.append(merged_episode)
141142

142143
return merged_episodes
143144

tests/test_merge.py

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,20 @@ def test_merge_two_datasets(dataset_a, dataset_b, tmp_path):
5656
assert (output / "episodes" / str(i)).is_dir()
5757

5858

59+
def test_merge_preserves_valid_flags(dataset_a, dataset_b, tmp_path):
60+
meta_a = _load_meta(dataset_a)
61+
for episode, valid in zip(meta_a["episodes"], (False, True)):
62+
episode["valid"] = valid
63+
with open(dataset_a / "metadata.yaml", "w") as f:
64+
yaml.safe_dump(meta_a, f)
65+
66+
output = tmp_path / "merged"
67+
merge_datasets([dataset_a, dataset_b], output)
68+
69+
meta = _load_meta(output)
70+
assert [ep.get("valid") for ep in meta["episodes"]] == [False, True, None, None]
71+
72+
5973
def test_merge_preserves_metadata_from_first_dataset(dataset_a, dataset_b, tmp_path):
6074
meta_a = _load_meta(dataset_a)
6175
output = tmp_path / "merged"

0 commit comments

Comments
 (0)