rough IFEval implementation using llm_instruction benchmark

Author: farook-edev

flutter/cpp/binary/BUILD

@@ -54,6 +54,7 @@ cc_binary(
         "//flutter/cpp/datasets:ade20k",
         "//flutter/cpp/datasets:coco",
         "//flutter/cpp/datasets:coco_gen",
+        "//flutter/cpp/datasets:ifeval",
         "//flutter/cpp/datasets:imagenet",
         "//flutter/cpp/datasets:mmlu_gen",
         "//flutter/cpp/datasets:snu_sr",

flutter/cpp/binary/main.cc

@@ -25,6 +25,7 @@ limitations under the License.
 #include "flutter/cpp/datasets/ade20k.h"
 #include "flutter/cpp/datasets/coco.h"
 #include "flutter/cpp/datasets/coco_gen.h"
+#include "flutter/cpp/datasets/ifeval.h"
 #include "flutter/cpp/datasets/imagenet.h"
 #include "flutter/cpp/datasets/mmlu_gen.h"
 #include "flutter/cpp/datasets/snu_sr.h"
@@ -70,6 +71,8 @@ DatasetConfig::DatasetType Str2DatasetType(absl::string_view name) {
     return DatasetConfig::COCOGEN;
   } else if (absl::EqualsIgnoreCase(name, "MMLU")) {
     return DatasetConfig::MMLU;
+  } else if (absl::EqualsIgnoreCase(name, "IFEVAL")) {
+    return DatasetConfig::IFEVAL;
   } else if (absl::EqualsIgnoreCase(name, "DUMMY")) {
     return DatasetConfig::NONE;
   } else {
@@ -91,6 +94,8 @@ DatasetConfig::DatasetType BenchmarkId2DatasetType(absl::string_view name) {
     return DatasetConfig::SNUSR;
   } else if (absl::StartsWith(name, "stable_diffusion")) {
     return DatasetConfig::COCOGEN;
+  } else if (absl::StartsWith(name, "llm_instruction")) {
+    return DatasetConfig::IFEVAL;
   } else if (absl::StartsWith(name, "llm")) {
     return DatasetConfig::MMLU;
   } else {
@@ -420,6 +425,32 @@ int Main(int argc, char *argv[]) {
       flag_list.insert(flag_list.end(), dataset_flags.begin(),
                        dataset_flags.end());
     } break;
+    case DatasetConfig::IFEVAL: {
+      bool loose_follow = false;
+      LOG(INFO) << "IFEval dataset for LLM benchmark";
+      std::string input_tfrecord, sp_path = "";
+      std::vector<Flag> dataset_flags{
+          Flag::CreateFlag(
+              "input_tfrecord", &input_tfrecord,
+              "Path to the tfrecord file containing inputs for the model.",
+              Flag::kRequired),
+          Flag::CreateFlag("sp_path", &sp_path,
+                           "Path to the sentencepiece model file.",
+                           Flag::kRequired),
+          Flag::CreateFlag("loose-follow", &loose_follow,
+                           "Whether to loosely check if the instructions are "
+                           "being followed"),
+      };
+
+      if (Flags::Parse(&argc, const_cast<const char **>(argv), dataset_flags) &&
+          backend) {
+        dataset.reset(
+            new IFEval(backend.get(), input_tfrecord, sp_path, loose_follow));
+      }
+      // Adds to flag_list for showing help.
+      flag_list.insert(flag_list.end(), dataset_flags.begin(),
+                       dataset_flags.end());
+    } break;
     case DatasetConfig::NONE:
     default:
       break;

flutter/cpp/datasets/BUILD

@@ -237,3 +237,37 @@ cc_library(
         "@org_tensorflow//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
     ],
 )
+
+cc_library(
+    name = "ifeval",
+    srcs = [
+        "ifeval.cc",
+    ],
+    hdrs = [
+        "ifeval.h",
+        "utils.h",
+    ],
+    copts = tflite_copts() + select({
+        "//flutter/android/commonlibs:use_asan": [
+            "-fsanitize=address",
+            "-g",
+            "-O1",
+            "-fno-omit-frame-pointer",
+        ],
+        "//conditions:default": [],
+    }),
+    deps = [
+        ":allocator",
+        "//flutter/cpp:mlperf_driver",
+        "//flutter/cpp:utils",
+        "//flutter/cpp/backends:external",
+        "//flutter/cpp/datasets/ifeval_utils",
+        "//flutter/cpp/datasets/mmlu_utils",
+        "//flutter/cpp/datasets/squad_utils",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_protobuf//:protobuf",
+        "@com_google_sentencepiece//:sentencepiece_processor",
+        "@org_tensorflow//tensorflow/lite/tools/evaluation:utils",
+        "@org_tensorflow//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
+    ],
+)