LLM pipeline implementation (#1040)

* WIP LLM pipeline and dataset implementation

* fixed issues preventing libraries from compiling, runtime errors not included

* upgrade TensorFlow to 2.18.0

* upgraded llm pipeline to use TFLite C++ api + small bug fixes

* basic flutter app support for icon and dataset

* added linux x86_64 config for internal testing

* updated bazel config to use SSE/MMX instructions

* fixed incorrect answer format and compression

* got pipeline and dataset to produce proper results + fixed issues where pipeline cannot handle an input size larger than the max prefill size

* added support for loadgen's token based performance measurement + implemented performance benchmark for LLM pipeline

* fixed bugs in inference process, first token function now handles only input and issue_query only handles output tokens

* optimized tensor retrieval for inference + added check for input size vs KV cache size

* clang-format

* mmlu dataset cleanup and formatting

* slight code cleanup

* fixed issue with genai ops import

* code/config cleanup

* add zero-shot option to MMLU constructor

* use function to detect which token is answer letter

* quick initial implementation of first token callback

* moved tokenizer to dataset side (possibly needs cleanup)

* added files needed for MMLU utils

* clang-format

* continued formatting

* code cleanup / issue_query signature update to vendor backends

* signature update for QTI/Samsung backends

* format

* formatted clang and bazel using docker based formatter

* reverted issue_query change for samsung + bazel formatting

* fix for MSVC C7555 error

* rough IFEval implementation using llm_instruction benchmark

* disabled XNNPACK AVX-VNNI for windows due to C2440 error

* moved accuracy calculation away from ProcessOutput, ifeval accuracy is calculated per instruction not per sample

* fixed issue with app not finding model/tokenizer

* properly format 0-shot prompts + allow for file/directory for model path

* formatting

* potential fix for windows C2440

* fix for aligned free for windows

* potential fix for IOS / windows CI issues

* ifeval check cleanup and bugfixes

* formatting

* all possible configs for removing eigen exceptions

* removed objc opts

* use token latencies in app

* enable exceptions for IOS

* disable FP16 AVX for x86 simulator

* attempt to enable exceptions for eigen

* 2nd attempt at enabling exceptions for IOS eigen

* fixed fexceptions syntax

* kitchen-sink approach to enable exceptions for IOS

* attempt to undefine EIGEN_EXCEPTIONS for IOS

* add global ovverride to disable eigen exceptions

* use ARM based macos for IOS build

* fixed and re-enabled eigen patch

* further fix for eigen patch

* even more patch fixing

* fixed typo in eigen patch

* fixed incorrect count in eigen patch

* force arm64 ios build

* use ARM64 simulator for IOS build

* use arm64 simulator for tflite on IOS

* set ios cpu argument for cpuinfo

* remvoed ios_sim prefix

* attempt at using arm64 simulator for IOS instead of x86

* attempt to force flutter to build ITs for arm64 only

* force arm64 for pods

* disable f16 instead of building for arm64

* more bazel config lines to disable fp16

* removed unavailable compiler flags

* provide patched fp16 lib with math workaround

* typo

* added patch arg

* created a math workaround patch compatible with fp16 version used by xnnpack

* datasets now provide token limits as inputs to pipeline

---------

Co-authored-by: Farook Al-Sammarraie <[email protected]>

Author: mohitmundhragithub

.bazelrc

@@ -10,6 +10,10 @@ build --spawn_strategy=standalone
 # This flag is required by tensorflow
 common --experimental_repo_remote_exec
 
+# Without these, tensorflow complains about lack of CUDA library.
+common --repo_env=TF_NEED_CUDA=0
+common --repo_env=TF_NEED_ROCM=0
+
 # Default options should come above this line.
 
 # Configure logs
@@ -18,13 +22,16 @@ build:verbose_logs --output_filter=
 
 # Suppress C++ compiler warnings, otherwise build logs become 10s of MBs.
 build:android --copt=-w
+build:linux --copt=-w
 build:ios --copt=-w
 build:windows --copt=/W0
 
 # Build in C++ 17 mode.
 build --cxxopt=-std=c++17
 build:android --cxxopt=-std=c++17
 build:android --host_cxxopt=-std=c++17
+build:linux --cxxopt=-std=c++17
+build:linux --host_cxxopt=-std=c++17
 build:ios --cxxopt=-std=c++17
 build:ios --host_cxxopt=-std=c++17
 build:ios --cxxopt=-xobjective-c++
@@ -41,10 +48,39 @@ build:android_x86_64 --config=android
 build:android_x86_64 --cpu=x86_64
 build:android_x86_64 --fat_apk_cpu=x86_64
 
+
+build:android_x86_64 --define=xnn_enable_avx512fp16=false
+build:android_x86_64 --define=xnn_enable_avxvnniint8=false
+
+# Linux configs
+build:linux_x86_64 --config=linux
+build:linux_x86_64 --cpu=k8
+# Not required, but enables the proper SSE/MMX instructions per CPU
+build:linux_x86_64 --copt=-march=native
+
+# These may be neccessary depending on CPU instruction support
+#build:linux_x86_64 --define=xnn_enable_avx=false
+#build:linux_x86_64 --define=xnn_enable_avx2=false
+#build:linux_x86_64 --define=xnn_enable_avx512=false
+build:linux_x86_64 --define=xnn_enable_avx512fp16=false
+#build:linux_x86_64 --define=xnn_enable_avxvnni=false
+build:linux_x86_64 --define=xnn_enable_avxvnniint8=false
+#build:linux_x86_64 --define=xnn_enable_vnni=false
+
+
+# Optional, enable for debugging or compilation errors
+#build:linux_x86_64 --action_env=CC=gcc
+#build:linux_x86_64 --action_env=CXX=g++
+#build:linux_x86_64 --strip=never
+#build:linux_x86_64 --copt=-fno-omit-frame-pointer
+#build:linux_x86_64 --linkopt=-fno-omit-frame-pointer
+
 # iOS configs
 build:ios --apple_platform_type=ios
 build:ios --copt=-Wno-c++11-narrowing
 build:ios --cxxopt=-fobjc-arc
+build:ios --copt=-DEIGEN_NOEXCEPTIONS_OVERRIDE
+build:ios --cxxopt=-DEIGEN_NOEXCEPTIONS_OVERRIDE
 
 # Windows configs
 
@@ -73,6 +109,10 @@ build:windows --host_linkopt=/OPT:REF
 build:windows --linkopt=/OPT:ICF
 build:windows --host_linkopt=/OPT:ICF
 
+# MSVC does not support XNNPACK AVXVNNI instructions (causes C2440 error).
+build:windows --define=xnn_enable_avxvnni=false
+build:windows --define=xnn_enable_avxvnniint8=false
+
 # Address sanitizer
 build:asan --strip=never
 build:asan --copt -fsanitize=address

WORKSPACE

@@ -48,6 +48,18 @@ http_archive(
 
 load("@tf_patch_finder//:patch_win_arm64.bzl", "PATCH_FILE")
 
+http_archive(
+    name = "FP16",
+    build_file = "@//third_party:FP16.BUILD",
+    patch_args = ["-p1"],
+    patches = ["//patches:fp16_math_workaround.patch"],
+    sha256 = "e66e65515fa09927b348d3d584c68be4215cfe664100d01c9dbc7655a5716d70",
+    strip_prefix = "FP16-0a92994d729ff76a58f692d3028ca1b64b145d91",
+    urls = [
+        "https://github.com/Maratyszcza/FP16/archive/0a92994d729ff76a58f692d3028ca1b64b145d91.zip",
+    ],
+)
+
 http_archive(
     name = "org_tensorflow",
     patch_args = ["-p1"],
@@ -59,13 +71,43 @@ http_archive(
         # Fix tensorflow not being able to read image files on Windows
         "//:flutter/third_party/tensorflow-fix-file-opening-mode-for-Windows.patch",
         "//:flutter/third_party/tf-eigen.patch",
-        # NDK 25 support
-        "//patches:ndk_25_r14.diff",
     ] + PATCH_FILE,
-    sha256 = "ce357fd0728f0d1b0831d1653f475591662ec5bca736a94ff789e6b1944df19f",
-    strip_prefix = "tensorflow-2.14.0",
+    sha256 = "d7876f4bb0235cac60eb6316392a7c48676729860da1ab659fb440379ad5186d",
+    strip_prefix = "tensorflow-2.18.0",
+    urls = [
+        "https://github.com/tensorflow/tensorflow/archive/v2.18.0.tar.gz",
+    ],
+)
+
+load("@org_tensorflow//third_party/gpus:cuda_configure.bzl", "cuda_configure")
+
+cuda_configure(name = "local_config_cuda")
+
+load("@org_tensorflow//third_party/gpus:rocm_configure.bzl", "rocm_configure")
+
+rocm_configure(name = "local_config_rocm")
+
+http_archive(
+    name = "com_google_sentencepiece",
+    build_file = "@//patches:sentencepiece.BUILD",
+    patch_args = ["-p1"],
+    patches = ["@//patches:com_google_sentencepiece.diff"],
+    sha256 = "8409b0126ebd62b256c685d5757150cf7fcb2b92a2f2b98efb3f38fc36719754",
+    strip_prefix = "sentencepiece-0.1.96",
+    urls = [
+        "https://github.com/google/sentencepiece/archive/refs/tags/v0.1.96.zip",
+    ],
+)
+
+http_archive(
+    name = "darts_clone",
+    build_file = "@//patches:darts_clone.BUILD",
+    patch_args = ["-p0"],
+    patches = ["//patches:darts_no_exceptions.diff"],
+    sha256 = "c97f55d05c98da6fcaf7f9ecc6a6dc6bc5b18b8564465f77abff8879d446491c",
+    strip_prefix = "darts-clone-e40ce4627526985a7767444b6ed6893ab6ff8983",
     urls = [
-        "https://github.com/tensorflow/tensorflow/archive/v2.14.0.tar.gz",
+        "https://github.com/s-yata/darts-clone/archive/e40ce4627526985a7767444b6ed6893ab6ff8983.zip",
     ],
 )
 

flutter/android/android-docker.mk

@@ -19,7 +19,7 @@ user_id=$(shell id -u)
 .PHONY: flutter/android/docker/image
 flutter/android/docker/image: output/docker/mlperf_mobile_flutter_android_${user_id}.stamp
 output/docker/mlperf_mobile_flutter_android_${user_id}.stamp: flutter/android/docker/Dockerfile
-	docker image build -t ${DOCKER_IMAGE_TAG} flutter/android/docker
+	DOCKER_BUILDKIT=1 docker buildx build --tag ${DOCKER_IMAGE_TAG} flutter/android/docker
 	mkdir -p output/docker
 	touch $@
 
@@ -68,4 +68,4 @@ docker/flutter/android/release: flutter/check-release-env flutter/android/docker
 docker/flutter/clean: flutter/check-release-env
 	MSYS2_ARG_CONV_EXCL="*" docker run \
 		${flutter_common_docker_flags} \
-		make flutter/clean
+		make flutter/clean

flutter/assets/icons/ic_task_llm.svg

@@ -336,6 +336,63 @@ task {
   }
 }
 
+task {
+  id: "llm"
+  name: "LLM"
+  max_throughput: 2000
+  max_accuracy: 1.0
+  scenario: "SingleStream"
+  runs {
+    normal {
+      min_query_count: 100
+      min_duration: 60
+      max_duration: 300
+    }
+    quick {
+      min_query_count: 10
+      min_duration: 10
+      max_duration: 40
+    }
+    rapid {
+      min_query_count: 6
+      min_duration: 6
+      max_duration: 60
+    }
+  }
+  datasets {
+    type: MMLU
+      full {
+      name: "TinyMMLU prompt set for LLM"
+      input_path: "local:///mlperf_datasets/tinymmlu/data.tfrecord"
+      input_checksum: "c20f9115582217af15e4d9955b41ace1"
+      groundtruth_path: ""
+      groundtruth_checksum: ""
+    }
+    lite {
+      name: "TinyMMLU prompt set for LLM"
+      input_path: "local:///mlperf_datasets/tinymmlu/data.tfrecord"
+      input_checksum: "c20f9115582217af15e4d9955b41ace1"
+      groundtruth_path: ""
+      groundtruth_checksum: ""
+    }
+    tiny {
+      name: "TinyMMLU prompt set for LLM"
+      input_path: "local:///mlperf_datasets/tinymmlu/data.tfrecord"
+      input_checksum: "c20f9115582217af15e4d9955b41ace1"
+      groundtruth_path: ""
+      groundtruth_checksum: ""
+    }
+  }
+  model {
+    id: "LLM"
+    name: "LLM"
+  }
+  custom_config {
+    id: "llm_tokenizer_path"
+    value: "llama3_1b.spm.model"
+  }
+}
+
 task {
   id: "stable_diffusion"
   name: "Stable Diffusion"

flutter/assets/icons/ic_task_llm_white.svg

@@ -45,7 +45,9 @@ class Backend {
   virtual const std::string& AcceleratorName() const = 0;
 
   // Run inference for a sample. Inputs is already set by SetInputs.
-  virtual void IssueQuery() = 0;
+  // TODO might be good to provide the callback and context along with the
+  // inputs if possible
+  virtual void IssueQuery(ft_callback callback, void* context) = 0;
 
   // Flush the staged queries immediately.
   virtual void FlushQueries() = 0;

flutter/assets/tasks.pbtxt

@@ -47,8 +47,8 @@ struct BackendFunctions {
   using AcceleratorNamePtr =
       std::add_pointer<const char*(mlperf_backend_ptr_t)>::type;
   using BackendDeletePtr = std::add_pointer<void(mlperf_backend_ptr_t)>::type;
-  using IssueQueryPtr =
-      std::add_pointer<mlperf_status_t(mlperf_backend_ptr_t)>::type;
+  using IssueQueryPtr = std::add_pointer<mlperf_status_t(
+      mlperf_backend_ptr_t, ft_callback, void*)>::type;
   using FlushQueriesPtr =
       std::add_pointer<mlperf_status_t(mlperf_backend_ptr_t)>::type;
 
@@ -157,8 +157,9 @@ class ExternalBackend : public Backend {
   }
 
   // Run inference for a sample.
-  void IssueQuery() override {
-    if (backend_functions_.issue_query(backend_ptr_) != MLPERF_SUCCESS) {
+  void IssueQuery(ft_callback callback, void* context) override {
+    if (backend_functions_.issue_query(backend_ptr_, callback, context) !=
+        MLPERF_SUCCESS) {
       LOG(FATAL) << "Error while inferencing model";
     }
   }

flutter/cpp/backend.h

@@ -54,7 +54,9 @@ cc_binary(
         "//flutter/cpp/datasets:ade20k",
         "//flutter/cpp/datasets:coco",
         "//flutter/cpp/datasets:coco_gen",
+        "//flutter/cpp/datasets:ifeval",
         "//flutter/cpp/datasets:imagenet",
+        "//flutter/cpp/datasets:mmlu_gen",
         "//flutter/cpp/datasets:snu_sr",
         "//flutter/cpp/datasets:squad",
         "//flutter/cpp/proto:mlperf_task_cc_proto",

flutter/cpp/backends/external.h

@@ -17,4 +17,10 @@
 docker/cmdline/android/release: flutter/android/docker/image
 	MSYS2_ARG_CONV_EXCL="*" docker run \
 		${flutter_common_docker_flags} \
-		make cmdline/android/bins/release
+		make cmdline/android/bins/release
+
+.PHONY: docker/cmdline/linux/release
+docker/cmdline/linux/release: flutter/android/docker/image
+	MSYS2_ARG_CONV_EXCL="*" docker run \
+		${flutter_common_docker_flags} \
+		make cmdline/linux/bins/release