Feat. Benchmark Sets

flutter/assets/tasks.pbtxt

@@ -1,9 +1,29 @@
 # proto-file: flutter/cpp/proto/mlperf_task.proto
 # proto-message: MLPerfConfig
 
-task {
+task_set {
   id: "image_classification_v2"
   name: "Image Classification v2"
+  option_set {
+    id: "options"
+    name: "Options"
+    opt {
+      id: "offline"
+      name: "Offline"
+    }
+    opt {
+      id: "online"
+      name: "Online"
+      enabled: true
+    }
+  }
+}
+
+task {
+  id: "image_classification_online_v2"
+  name: "Image Classification v2 (Online)"
+  task_set: "image_classification_v2"
+  required_option: "online"
   max_throughput: 1000
   max_accuracy: 1.0
   scenario: "SingleStream"
@@ -282,6 +302,8 @@ task {
 task {
   id: "image_classification_offline_v2"
   name: "Image Classification v2 (Offline)"
+  task_set: "image_classification_v2"
+  required_option: "offline"
   max_throughput: 2000
   max_accuracy: 1.0
   scenario: "Offline"

flutter/cpp/proto/mlperf_task.proto

@@ -22,16 +22,17 @@ option java_package = "org.mlperf.proto";
 
 // App config
 //
-// Next ID: 2
+// Next ID: 3
 message MLPerfConfig {
   // All tasks will be run.
   repeated TaskConfig task = 1;
+  repeated TaskSet task_set = 2;
 }
 
 // Config of the mlperf tasks.
 // A task is basically a combination of models and a dataset.
 //
-// Next ID: 13
+// Next ID: 15
 message TaskConfig {
   // Must be unique in one task file. Ex: image_classification
   // used to match backend settings
@@ -48,6 +49,37 @@ message TaskConfig {
   required DatasetConfig datasets = 8;
   required ModelConfig model = 9;
   repeated CustomConfig custom_config = 11;
+  optional string task_set = 13;
+  repeated string required_option = 14;
+}
+
+// A group to control multiple tasks through the use of options.
+//
+// Next ID: 4
+message TaskSet {
+  required string id = 1;
+  required string name = 2;
+  repeated OptionSet option_set = 3;
+}
+
+// A container for similar options for a task-set
+//
+// Next ID: 6
+message OptionSet {
+  required string id = 1;
+  required string name = 2;
+  optional int32 min_selected = 3 [default = 0];
+  optional int32 max_selected = 4 [default = 0];
+  repeated Option opt = 5;
+}
+
+// A fancy boolean with a name and an id.
+//
+// Next ID: 4
+message Option {
+  required string id = 1;
+  required string name = 2;
+  optional bool enabled = 3 [default = false];
 }
 
 // Run configurations

flutter/lib/app_constants.dart

@@ -22,7 +22,7 @@ class BenchmarkId {
   static const imageSegmentationV2 = 'image_segmentation_v2';
   static const naturalLanguageProcessing = 'natural_language_processing';
   static const superResolution = 'super_resolution';
-  static const imageClassificationV2 = 'image_classification_v2';
+  static const imageClassificationV2 = 'image_classification_online_v2';
   static const imageClassificationOfflineV2 = 'image_classification_offline_v2';
   static const stableDiffusion = 'stable_diffusion';
   static const llm = 'llm';

flutter/lib/benchmark/benchmark.dart

@@ -116,13 +116,129 @@ class Benchmark {
   }
 }
 
+class BenchmarkSet {
+  final pb.TaskSet config;
+  late final List<Benchmark> benchmarks;
+  late final List<BenchmarkOptionSet> optionSets;
+  //TODO use pass by reference
+  late final Map<String, int> optionMap;
+
+  BenchmarkSet(
+      {required this.config, List<Benchmark> allBenchmarks = const []}) {
+    optionSets =
+        config.optionSet.map((e) => BenchmarkOptionSet(config: e)).toList();
+    benchmarks =
+        allBenchmarks.where((e) => e.taskConfig.taskSet == config.id).toList();
+    optionMap = {
+      for (final (index, item) in optionSets.indexed)
+        for (final key in item.options.keys) key: index,
+    };
+    applyOptions();
+  }
+
+  void applyOptions() {
+    for (Benchmark benchmark in benchmarks) {
+      benchmark.isActive = true;
+      for (String optionId in benchmark.taskConfig.requiredOption) {
+        final index = optionMap[optionId];
+        final optionSet =
+            (index != null && index >= 0 && index < optionSets.length)
+                ? optionSets[index]
+                : null;
+        if (!(optionSet?.getOption(optionId) ?? false)) {
+          benchmark.isActive = false;
+          break;
+        }
+      }
+    }
+  }
+}
+
+class BenchmarkOptionSet {
+  final pb.OptionSet config;
+  late final Map<String, BenchmarkOption> options;
+  late int selected;
+
+  BenchmarkOptionSet({required this.config}) {
+    options = _createOptions(config);
+    selected = options.values.where((e) => e.enabled).length;
+    if ((config.maxSelected > 0 && selected > config.maxSelected) ||
+        (config.minSelected > 0 && selected < config.minSelected)) {
+      throw Exception('pbtxt config error! Option constraint mismatch!');
+    }
+  }
+
+  Map<String, BenchmarkOption> _createOptions(pb.OptionSet config) {
+    return {
+      for (final item in config.opt) item.id: BenchmarkOption(config: item)
+    };
+  }
+
+  bool? getOption(String id) {
+    return options[id]?.enabled;
+  }
+
+  bool setOptionTo(String id, bool value) {
+    BenchmarkOption? opt = options[id];
+    if (opt == null) return false;
+    if (!value) return unsetOption(id);
+    return setOption(id);
+  }
+
+  bool setOption(String id) {
+    BenchmarkOption? opt = options[id];
+    if (opt == null ||
+        (config.maxSelected > 0 && selected == config.maxSelected)) {
+      return false;
+    }
+    opt.enabled = true;
+    return true;
+  }
+
+  bool unsetOption(String id) {
+    BenchmarkOption? opt = options[id];
+    if (opt == null ||
+        (config.minSelected > 0 && selected == config.minSelected)) {
+      return false;
+    }
+    opt.enabled = false;
+    return true;
+  }
+
+  bool toggleOption(String id) {
+    BenchmarkOption? opt = options[id];
+    if (opt == null) return false;
+    if (opt.enabled) return unsetOption(id);
+    return setOption(id);
+  }
+}
+
+class BenchmarkOption {
+  final pb.Option config;
+  late bool enabled;
+
+  BenchmarkOption({required this.config}) {
+    enabled = config.enabled;
+  }
+
+  String get id => config.id;
+
+  String get name => config.name;
+}
+
 class BenchmarkStore {
+  // NOTE this includes benchmarks inside sets
   final List<Benchmark> allBenchmarks = <Benchmark>[];
+  final List<BenchmarkSet> benchmarkSets = <BenchmarkSet>[];
 
   List<Benchmark> get activeBenchmarks {
     return allBenchmarks.where((e) => e.isActive).toList();
   }
 
+  List<Benchmark> get looseBenchmarks {
+    return allBenchmarks.where((e) => e.taskConfig.taskSet.isEmpty).toList();
+  }
+
   BenchmarkStore({
     required pb.MLPerfConfig appConfig,
     required List<pb.BenchmarkSetting> backendConfig,
@@ -147,6 +263,10 @@ class BenchmarkStore {
         isActive: enabled,
       ));
     }
+    for (final setConfig in appConfig.taskSet) {
+      benchmarkSets
+          .add(BenchmarkSet(config: setConfig, allBenchmarks: allBenchmarks));
+    }
   }
 
   List<Resource> listResources({

flutter/lib/benchmark/info.dart

@@ -27,6 +27,12 @@ class BenchmarkInfo {
 
   BenchmarkLocalizationInfo getLocalizedInfo(AppLocalizations stringResources) {
     switch (task.id) {
+      case (BenchmarkId.llm):
+        // TODO translate this and add proper info
+        return BenchmarkLocalizationInfo(
+            name: 'LLM',
+            detailsTitle: 'Large Language Model',
+            detailsContent: 'you know what ChatGPT is...');
       case (BenchmarkId.imageClassificationV2):
         return BenchmarkLocalizationInfo(
           name: stringResources.benchNameImageClassification,

flutter/lib/benchmark/state.dart

@@ -77,6 +77,10 @@ class BenchmarkState extends ChangeNotifier {
 
   List<Benchmark> get allBenchmarks => _benchmarkStore.allBenchmarks;
 
+  List<Benchmark> get looseBenchmarks => _benchmarkStore.looseBenchmarks;
+
+  List<BenchmarkSet> get benchmarkSets => _benchmarkStore.benchmarkSets;
+
   List<Benchmark> get activeBenchmarks => _benchmarkStore.activeBenchmarks;
 
   late BenchmarkStore _benchmarkStore;
@@ -348,6 +352,39 @@ class BenchmarkState extends ChangeNotifier {
     notifyListeners();
   }
 
+  void benchmarkSetOption(
+      BenchmarkSet benchmarkSet, String option, bool value) {
+    benchmarkSet.optionSets[benchmarkSet.optionMap[option]!]
+        .setOptionTo(option, value);
+    benchmarkSet.applyOptions();
+    notifyListeners();
+  }
+
+// Config section controls
+  final Set<BenchmarkSet> _openAdvancedSets = {};
+  bool isAdvancedConfigOpen(BenchmarkSet set) =>
+      _openAdvancedSets.contains(set);
+  void toggleAdvancedConfig(BenchmarkSet set) {
+    _openAdvancedSets.contains(set)
+        ? _openAdvancedSets.remove(set)
+        : _openAdvancedSets.add(set);
+    notifyListeners();
+  }
+
+  final Set<String> _expandedOptionSets = {};
+
+  bool isOptionsExpanded(BenchmarkSet bset) =>
+      _expandedOptionSets.contains(bset.config.id);
+
+  void toggleOptionsExpanded(BenchmarkSet bset) {
+    if (_expandedOptionSets.contains(bset.config.id)) {
+      _expandedOptionSets.remove(bset.config.id);
+    } else {
+      _expandedOptionSets.add(bset.config.id);
+    }
+    notifyListeners();
+  }
+
   @override
   void notifyListeners() {
     super.notifyListeners();

flutter/lib/ui/error_dialog.dart

@@ -120,7 +120,7 @@ Future<void> showErrorDialog(
 
 Future<void> showResourceMissingDialog(
     BuildContext context, List<String> messages,
-    {Benchmark? benchmark}) async {
+    {Benchmark? benchmark, BenchmarkSet? benchmarkSet}) async {
   final l10n = AppLocalizations.of(context)!;
 
   Icon icon = const Icon(Icons.warning_amber_rounded,
@@ -167,8 +167,11 @@ Future<void> showResourceMissingDialog(
               Text(benchmark != null
                   ? l10n.dialogContentMissingFiles
                       .replaceAll('<name>', benchmark.info.taskName)
-                  : l10n.dialogContentMissingFiles
-                      .replaceAll('<name>', l10n.benchNameVarious)),
+                  : benchmarkSet != null
+                      ? l10n.dialogContentMissingFiles
+                          .replaceAll('<name>', benchmarkSet.config.name)
+                      : l10n.dialogContentMissingFiles
+                          .replaceAll('<name>', l10n.benchNameVarious)),
             ],
           ),
         ),
@@ -195,7 +198,11 @@ Future<void> showResourceMissingDialog(
                       MaterialPageRoute(
                         builder: (context) => ResourcesScreen(
                           autoStart: true,
-                          singleBenchmarkDownload: benchmark,
+                          benchmarksToDownload: (benchmark != null
+                              ? <Benchmark>[benchmark]
+                              : benchmarkSet?.benchmarks
+                                  .where((e) => e.isActive)
+                                  .toList()),
                         ),
                       ),
                     );