feat: add checksum verification for datasets (#939)

* Add input_checksum and groundtruth_checksum

* Verify datasets checksum in app

Author: anhappdev

flutter/assets/tasks.pbtxt

@@ -15,17 +15,23 @@ task {
     full {
       name: "Imagenet classification validation set"
       input_path: "local:///mlperf_datasets/imagenet/img"
+      input_checksum: ""
       groundtruth_path: "local:///mlperf_datasets/imagenet/imagenet_val_full.txt"
+      groundtruth_checksum: ""
     }
     lite {
       name: "Open images subset for classification"
       input_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v0_7/imagenet.zip"
+      input_checksum: "fdcce28e06475321dc9b1c63a5539a4e"
       groundtruth_path: ""
+      groundtruth_checksum: ""
     }
     tiny {
       name: "Imagenet dataset for integration test"
       input_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v0_7/imagenet_tiny.zip"
+      input_checksum: "42cb83d80d0341d6719d68b32322adf2"
       groundtruth_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v3_0/imagenet_tiny-groundtruth.txt"
+      groundtruth_checksum: "cc6d83ca25daeb2475f36c079e53bc26"
     }
   }
   model {
@@ -52,17 +58,23 @@ task {
     full {
       name: "COCO 2017 validation set"
       input_path: "local:///mlperf_datasets/coco/img"
+      input_checksum: ""
       groundtruth_path: "local:///mlperf_datasets/coco/coco_val_full.pbtxt"
+      groundtruth_checksum: ""
     }
     lite {
       name: "Open images subset for detection"
       input_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v0_7/coco.zip"
+      input_checksum: "c630e2ddf96a1e9482c4a9e1d4aced70"
       groundtruth_path: ""
+      groundtruth_checksum: ""
     }
     tiny {
       name: "Coco dataset for integration test"
       input_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v3_0/coco-test.zip"
+      input_checksum: "7440da62aee7043b825d3ac1c6732e0a"
       groundtruth_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v1_0/coco_val_tiny.pbtxt"
+      groundtruth_checksum: "0bae0c934cae7fa41f178c085bf31c7b"
     }
   }
   model {
@@ -89,17 +101,23 @@ task {
     full {
       name: "ADE20K validation set"
       input_path: "local:///mlperf_datasets/ade20k/images"
+      input_checksum: ""
       groundtruth_path: "local:///mlperf_datasets/ade20k/annotations"
+      groundtruth_checksum: ""
     }
     lite {
       name: "Open images subset for segmentation"
       input_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v0_7/ade20k.zip"
+      input_checksum: "97f48388e5ca3b4f19047e0b7a062c88"
       groundtruth_path: ""
+      groundtruth_checksum: ""
     }
     tiny {
       name: "ADE20K validation subset for integration test"
       input_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v3_1/ade20k_tiny.zip"
+      input_checksum: "544a538eed16283586bd583dd39c44db"
       groundtruth_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v3_1/ade20k_tiny_annotations.zip"
+      groundtruth_checksum: "a0e4b79dceac96a784b7abaa59aa76df"
     }
   }
   model {
@@ -125,17 +143,23 @@ task {
     full {
       name: "Squad V1.1 validation set"
       input_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v1_0/squad_eval_mini.tfrecord"
+      input_checksum: "3c9a1c4c373037889938d4b13d41f4c2"
       groundtruth_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v1_0/squad_groundtruth.tfrecord"
+      groundtruth_checksum: "f9d6d209ec9a05a522749a6bd597a75c"
     }
     lite {
       name: "Squad V1.1 mini set"
       input_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v1_0/squad_eval_mini.tfrecord"
+      input_checksum: "3c9a1c4c373037889938d4b13d41f4c2"
       groundtruth_path: ""
+      groundtruth_checksum: ""
     }
     tiny {
       name: "Squad V1.1 mini set"
       input_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v1_0/squad_eval_tiny.tfrecord"
+      input_checksum: "fa018f599d0016ed385b31aee7b00bed"
       groundtruth_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v1_0/squad_groundtruth.tfrecord"
+      groundtruth_checksum: "f9d6d209ec9a05a522749a6bd597a75c"
     }
   }
   model {
@@ -158,17 +182,23 @@ task {
     full {
       name: "SNUSR dataset for accuracy test"
       input_path: "local:///mlperf_datasets/snusr/lr"
+      input_checksum: ""
       groundtruth_path: "local:///mlperf_datasets/snusr/hr"
+      groundtruth_checksum: ""
     }
     lite {
       name: "SNUSR dataset for performance test"
       input_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v3_1/snusr_lr_png.zip"
+      input_checksum: "38bf296e38cba0a8229d946fc1a5095a"
       groundtruth_path: ""
+      groundtruth_checksum: ""
     }
     tiny {
       name: "SNUSR dataset for integration test"
       input_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v3_1/snusr_lr_png_tiny.zip"
+      input_checksum: "134e9f96e4bbd5e925093e30a4433548"
       groundtruth_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v3_1/snusr_hr_png_tiny.zip"
+      groundtruth_checksum: "c8eac83e32d163552ca8e317c2dd57c3"
     }
   }
   model {
@@ -193,17 +223,23 @@ task {
     full {
       name: "Imagenet classification validation set"
       input_path: "local:///mlperf_datasets/imagenet/img"
+      input_checksum: ""
       groundtruth_path: "local:///mlperf_datasets/imagenet/imagenet_val_full.txt"
+      groundtruth_checksum: ""
     }
     lite {
       name: "Open images subset for classification"
       input_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v0_7/imagenet.zip"
+      input_checksum: "fdcce28e06475321dc9b1c63a5539a4e"
       groundtruth_path: ""
+      groundtruth_checksum: ""
     }
     tiny {
       name: "Imagenet dataset for integration test"
       input_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v0_7/imagenet_tiny.zip"
+      input_checksum: "42cb83d80d0341d6719d68b32322adf2"
       groundtruth_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v3_0/imagenet_tiny-groundtruth.txt"
+      groundtruth_checksum: "cc6d83ca25daeb2475f36c079e53bc26"
     }
   }
   model {
@@ -230,17 +266,23 @@ task {
     full {
       name: "COCO validation set for Stable Diffusion"
       input_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v4_1/coco_gen_test.tfrecord"
-      groundtruth_path: "local:///mlperf_models/stable-diffusion/clip_model_512x512.tflite"
+      input_checksum: "b564d2c228a867148fa7d6df415a0368"
+      groundtruth_path: "local:///mlperf_models/stable-diffusion/clip_model_512x512_openai-clip-vit-large-patch14.tflite"
+      groundtruth_checksum: "39a07ffaea0806ee6148874ef228cc77"
     }
     lite {
       name: "COCO validation set for Stable Diffusion"
       input_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v4_1/coco_gen_full.tfrecord"
+      input_checksum: "5cf967d2b2128edeb1b4d6eca6e8d94d"
       groundtruth_path: ""
+      groundtruth_checksum: ""
     }
     tiny {
       name: "COCO validation set for Stable Diffusion"
-      input_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v4_1/coco_gen.tfrecord"
-      groundtruth_path: "local:///mlperf_models/stable-diffusion/clip_model_512x512.tflite"
+      input_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v4_1/coco_gen_test.tfrecord"
+      input_checksum: "b564d2c228a867148fa7d6df415a0368"
+      groundtruth_path: "local:///mlperf_models/stable-diffusion/clip_model_512x512_openai-clip-vit-large-patch14.tflite"
+      groundtruth_checksum: "39a07ffaea0806ee6148874ef228cc77"
     }
   }
   model {

flutter/cpp/proto/mlperf_task.proto

@@ -85,10 +85,14 @@ message DatasetConfig {
 message OneDatasetConfig {
   // Human-readable name of the dataset
   required string name = 1;
-  // URL or local path to dataset input files
+  // URL or local path to dataset input file
   required string input_path = 2;
-  // URL or local path to dataset groundtruth files
+  // MD5 checksum to validate the input file
+  required string input_checksum = 4;
+  // URL or local path to dataset groundtruth file
   required string groundtruth_path = 3;
+  // MD5 checksum to validate the groundtruth file
+  required string groundtruth_checksum = 5;
 }
 
 // Config of a model.

flutter/lib/benchmark/benchmark.dart

@@ -162,12 +162,14 @@ class BenchmarkStore {
       for (var mode in modes) {
         final dataset = mode.chooseDataset(b.taskConfig);
         final data = Resource(
-          path: dataset.inputPath,
           type: ResourceTypeEnum.datasetData,
+          path: dataset.inputPath,
+          md5Checksum: dataset.inputChecksum,
         );
         final groundtruth = Resource(
-          path: dataset.groundtruthPath,
           type: ResourceTypeEnum.datasetGroundtruth,
+          path: dataset.groundtruthPath,
+          md5Checksum: dataset.groundtruthChecksum,
         );
         result.addAll([data, groundtruth]);
       }

flutter/lib/resources/resource.dart

@@ -1,14 +1,14 @@
 enum ResourceTypeEnum { model, datasetData, datasetGroundtruth }
 
 class Resource {
-  final String path;
   final ResourceTypeEnum type;
-  final String? md5Checksum;
+  final String path;
+  final String md5Checksum;
 
   Resource({
-    required this.path,
     required this.type,
-    this.md5Checksum,
+    required this.path,
+    required this.md5Checksum,
   });
 
   @override

flutter/lib/resources/resource_manager.dart

@@ -200,7 +200,7 @@ class ResourceManager {
     final checksumFailedResources = <Resource>[];
     for (final resource in resources) {
       final md5Checksum = resource.md5Checksum;
-      if (md5Checksum == null || md5Checksum.isEmpty) continue;
+      if (md5Checksum.isEmpty) continue;
       String? localPath;
       if (cacheManager.isResourceAnArchive(resource.path)) {
         localPath = cacheManager.getArchive(resource.path);

flutter/unit_test/benchmark/benchmark_store_test.dart

@@ -116,14 +116,16 @@ void main() {
       expect(
           resources,
           contains(Resource(
-            path: task1.datasets.full.inputPath,
             type: ResourceTypeEnum.datasetData,
+            path: task1.datasets.full.inputPath,
+            md5Checksum: task1.datasets.full.inputChecksum,
           )));
       expect(
           resources,
           contains(Resource(
-            path: task1.datasets.full.groundtruthPath,
             type: ResourceTypeEnum.datasetGroundtruth,
+            path: task1.datasets.full.groundtruthPath,
+            md5Checksum: task1.datasets.full.groundtruthChecksum,
           )));
       expect(
           resources,
@@ -153,8 +155,9 @@ void main() {
       expect(
           resources,
           contains(Resource(
-            path: task1.datasets.lite.inputPath,
             type: ResourceTypeEnum.datasetData,
+            path: task1.datasets.lite.inputPath,
+            md5Checksum: task1.datasets.lite.inputChecksum,
           )));
       expect(
           resources,
@@ -187,8 +190,9 @@ void main() {
       expect(
           resources,
           contains(Resource(
-            path: task1.datasets.tiny.inputPath,
             type: ResourceTypeEnum.datasetData,
+            path: task1.datasets.tiny.inputPath,
+            md5Checksum: task1.datasets.tiny.inputChecksum,
           )));
       expect(
           resources,