Update dataset paths and align benchmark criteria across tasks (#4262)

eugene123tw · web-flow · commit 5b94b7dc802a · 2025-02-27T11:15:26.000Z
diff --git a/tests/perf_v2/tasks/classification.py b/tests/perf_v2/tasks/classification.py
@@ -67,7 +67,7 @@
     ),
     DatasetInfo(
         name="multiclass_food20_large",
-        path=Path("multiclass_classification/multiclass_food20_large"),
+        path=Path("multiclass_classification/multilabel_food101_large"),
         group="large",
         extra_overrides={},
     ),
@@ -99,7 +99,7 @@
     ),
     DatasetInfo(
         name="multilabel_food20_large",
-        path=Path("multilabel_classification/multilabel_food20_large"),
+        path=Path("multilabel_classification/multilabel_food101_large"),
         group="large",
         extra_overrides={},
     ),
@@ -131,10 +131,4 @@
         group="medium",
         extra_overrides={},
     ),
-    DatasetInfo(
-        name="cifar100_label_group_datum_format_large",
-        path=Path("hlabel_classification/cifar100_label_group_datum_format_large"),
-        group="large",
-        extra_overrides={},
-    ),
 ]
diff --git a/tests/perf_v2/tasks/keypoint_detection.py b/tests/perf_v2/tasks/keypoint_detection.py
@@ -42,6 +42,7 @@
     ),
 ]
 
+# TODO (someone): align with detection task (adding gpu_mem, latency, optimize/e2e, etc)
 BENCHMARK_CRITERIA = [
     Criterion(name="train/epoch", summary="max", compare="<", margin=0.1),
     Criterion(name="train/e2e_time", summary="max", compare="<", margin=0.1),
diff --git a/tests/perf_v2/tasks/semantic_segmentation.py b/tests/perf_v2/tasks/semantic_segmentation.py
@@ -50,7 +50,7 @@
     ),
 ]
 
-# TODO (someone): Compare with DETECTION CRITERIA and fill in the missing values
+# TODO (someone): align with detection task (adding gpu_mem, latency, optimize/e2e, etc)
 BENCHMARK_CRITERIA = [
     Criterion(name="train/epoch", summary="max", compare="<", margin=0.1),
     Criterion(name="train/e2e_time", summary="max", compare="<", margin=0.1),
diff --git a/tests/perf_v2/tasks/visual_prompting.py b/tests/perf_v2/tasks/visual_prompting.py
@@ -46,7 +46,7 @@
 ]
 
 
-# TODO (someone): Compare with DETECTION CRITERIA and fill in the missing values
+# TODO (someone): align with detection task (adding gpu_mem, latency, optimize/e2e, etc)
 BENCHMARK_CRITERIA = [
     Criterion(name="train/epoch", summary="max", compare="<", margin=0.1),
     Criterion(name="train/e2e_time", summary="max", compare="<", margin=0.1),

Original file line number	Diff line number	Diff line change
`@@ -42,6 +42,7 @@`
`42`	`42`	`),`
`43`	`43`	`]`
`44`	`44`
	`45`	`+# TODO (someone): align with detection task (adding gpu_mem, latency, optimize/e2e, etc)`
`45`	`46`	`BENCHMARK_CRITERIA = [`
`46`	`47`	`Criterion(name="train/epoch", summary="max", compare="<", margin=0.1),`
`47`	`48`	`Criterion(name="train/e2e_time", summary="max", compare="<", margin=0.1),`
Original file line number	Diff line number	Diff line change
`@@ -50,7 +50,7 @@`
`50`	`50`	`),`
`51`	`51`	`]`
`52`	`52`
`53`		`-# TODO (someone): Compare with DETECTION CRITERIA and fill in the missing values`
	`53`	`+# TODO (someone): align with detection task (adding gpu_mem, latency, optimize/e2e, etc)`
`54`	`54`	`BENCHMARK_CRITERIA = [`
`55`	`55`	`Criterion(name="train/epoch", summary="max", compare="<", margin=0.1),`
`56`	`56`	`Criterion(name="train/e2e_time", summary="max", compare="<", margin=0.1),`
Original file line number	Diff line number	Diff line change
`@@ -46,7 +46,7 @@`
`46`	`46`	`]`
`47`	`47`
`48`	`48`
`49`		`-# TODO (someone): Compare with DETECTION CRITERIA and fill in the missing values`
	`49`	`+# TODO (someone): align with detection task (adding gpu_mem, latency, optimize/e2e, etc)`
`50`	`50`	`BENCHMARK_CRITERIA = [`
`51`	`51`	`Criterion(name="train/epoch", summary="max", compare="<", margin=0.1),`
`52`	`52`	`Criterion(name="train/e2e_time", summary="max", compare="<", margin=0.1),`