From 8d4e201be64e61aedaeed340c8f9c85fe3a119ee Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Mon, 9 Sep 2024 23:34:22 -0700 Subject: [PATCH 1/3] Try with any pte --- .../android-llm-device-farm-test-spec.yml | 40 ++++++++++++++----- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/extension/android/benchmark/android-llm-device-farm-test-spec.yml b/extension/android/benchmark/android-llm-device-farm-test-spec.yml index 4e3274ce66f..3feacee77fb 100644 --- a/extension/android/benchmark/android-llm-device-farm-test-spec.yml +++ b/extension/android/benchmark/android-llm-device-farm-test-spec.yml @@ -10,18 +10,20 @@ phases: commands: # Prepare the model and the tokenizer - adb -s $DEVICEFARM_DEVICE_UDID shell "ls -la /sdcard/" - - adb -s $DEVICEFARM_DEVICE_UDID shell "mkdir -p /data/local/tmp/llama/" - - adb -s $DEVICEFARM_DEVICE_UDID shell "mv /sdcard/*.bin /data/local/tmp/llama/" - - adb -s $DEVICEFARM_DEVICE_UDID shell "mv /sdcard/*.pte /data/local/tmp/llama/" - - adb -s $DEVICEFARM_DEVICE_UDID shell "chmod 664 /data/local/tmp/llama/*.bin" - - adb -s $DEVICEFARM_DEVICE_UDID shell "chmod 664 /data/local/tmp/llama/*.pte" - - adb -s $DEVICEFARM_DEVICE_UDID shell "ls -la /data/local/tmp/llama/" + - adb -s $DEVICEFARM_DEVICE_UDID shell "mkdir -p /data/local/tmp/minibench/" + - adb -s $DEVICEFARM_DEVICE_UDID shell "mv /sdcard/*.bin /data/local/tmp/minibench/" + - adb -s $DEVICEFARM_DEVICE_UDID shell "mv /sdcard/*.pte /data/local/tmp/minibench/" + - adb -s $DEVICEFARM_DEVICE_UDID shell "chmod 664 /data/local/tmp/minibench/*.bin" + - adb -s $DEVICEFARM_DEVICE_UDID shell "chmod 664 /data/local/tmp/minibench/*.pte" + - adb -s $DEVICEFARM_DEVICE_UDID shell "ls -la /data/local/tmp/minibench/" test: commands: # By default, the following ADB command is used by Device Farm to run your Instrumentation test. # Please refer to Android's documentation for more options on running instrumentation tests with adb: # https://developer.android.com/studio/test/command-line#run-tests-with-adb + + # Run the Instrumentation test for sanity check - echo "Starting the Instrumentation test" - | adb -s $DEVICEFARM_DEVICE_UDID shell "am instrument -r -w --no-window-animation \ @@ -67,15 +69,31 @@ phases: fi; # Run the new generic benchmark activity https://developer.android.com/tools/adb#am - - echo "Run LLM benchmark" + - echo "Determine model type" + - | + BIN_FOUND="$(adb -s $DEVICEFARM_DEVICE_UDID shell find /data/local/tmp/minibench/ -name '*.bin')" + if [ -z "$BIN_FOUND" ]; then + echo "No tokenizer files found in /data/local/tmp/minibench/" + else + echo "tokenizer files found in /data/local/tmp/minibench/" + fi + + - echo "Run benchmark" - | - adb -s $DEVICEFARM_DEVICE_UDID shell am start -W -n org.pytorch.minibench/.LlmBenchmarkActivity \ - --es "model_dir" "/data/local/tmp/llama" \ - --es "tokenizer_path" "/data/local/tmp/llama/tokenizer.bin" + adb -s $DEVICEFARM_DEVICE_UDID shell am force-stop org.pytorch.minibench + if [ -z "$BIN_FOUND" ]; then + adb -s $DEVICEFARM_DEVICE_UDID shell am start -W -n org.pytorch.minibench/.BenchmarkActivity \ + --es "model_dir" "/data/local/tmp/minibench" + else + adb -s $DEVICEFARM_DEVICE_UDID shell am start -W -n org.pytorch.minibench/.LlmBenchmarkActivity \ + --es "model_dir" "/data/local/tmp/minibench" \ + --es "tokenizer_path" "/data/local/tmp/minibench/tokenizer.bin" + fi + post_test: commands: - - echo "Gather LLM benchmark results" + - echo "Gather benchmark results" - | BENCHMARK_RESULTS="" ATTEMPT=0 From 5de41f0c8a22e2683f0e49b4287ac201f95024b1 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Wed, 11 Sep 2024 23:18:29 -0700 Subject: [PATCH 2/3] Modify minibench app to product json result --- .../android-llm-device-farm-test-spec.yml | 3 +- .../pytorch/minibench/BenchmarkActivity.java | 29 +++++++++++++++++-- .../minibench/LlmBenchmarkActivity.java | 20 ++++++------- 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/extension/android/benchmark/android-llm-device-farm-test-spec.yml b/extension/android/benchmark/android-llm-device-farm-test-spec.yml index 3feacee77fb..ffb528767a5 100644 --- a/extension/android/benchmark/android-llm-device-farm-test-spec.yml +++ b/extension/android/benchmark/android-llm-device-farm-test-spec.yml @@ -16,6 +16,7 @@ phases: - adb -s $DEVICEFARM_DEVICE_UDID shell "chmod 664 /data/local/tmp/minibench/*.bin" - adb -s $DEVICEFARM_DEVICE_UDID shell "chmod 664 /data/local/tmp/minibench/*.pte" - adb -s $DEVICEFARM_DEVICE_UDID shell "ls -la /data/local/tmp/minibench/" + - adb -s $DEVICEFARM_DEVICE_UDID shell "run-as org.pytorch.minibench rm -rf files" test: commands: @@ -95,7 +96,7 @@ phases: commands: - echo "Gather benchmark results" - | - BENCHMARK_RESULTS="" + BENCHMARK_RESULTS=$(adb -s $DEVICEFARM_DEVICE_UDID shell run-as org.pytorch.minibench cat files/benchmark_results.json) ATTEMPT=0 MAX_ATTEMPT=10 while [ -z "${BENCHMARK_RESULTS}" ] && [ $ATTEMPT -lt $MAX_ATTEMPT ]; do diff --git a/extension/android/benchmark/app/src/main/java/org/pytorch/minibench/BenchmarkActivity.java b/extension/android/benchmark/app/src/main/java/org/pytorch/minibench/BenchmarkActivity.java index a79f668f80b..9ede7d69184 100644 --- a/extension/android/benchmark/app/src/main/java/org/pytorch/minibench/BenchmarkActivity.java +++ b/extension/android/benchmark/app/src/main/java/org/pytorch/minibench/BenchmarkActivity.java @@ -11,10 +11,14 @@ import android.app.Activity; import android.content.Intent; import android.os.Bundle; +import com.google.gson.Gson; import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; import org.pytorch.executorch.Module; public class BenchmarkActivity extends Activity { @@ -32,20 +36,39 @@ protected void onCreate(Bundle savedInstanceState) { int numIter = intent.getIntExtra("num_iter", 10); // TODO: Format the string with a parsable format - StringBuilder resultText = new StringBuilder(); + Stats stats = new Stats(); Module module = Module.load(model.getPath()); for (int i = 0; i < numIter; i++) { long start = System.currentTimeMillis(); module.forward(); long forwardMs = System.currentTimeMillis() - start; - resultText.append(forwardMs).append(";"); + stats.latency.add(forwardMs); } + // TODO (huydhn): Remove txt files here once the JSON format is ready try (FileWriter writer = new FileWriter(getFilesDir() + "/benchmark_results.txt")) { - writer.write(resultText.toString()); + writer.write(stats.toString()); } catch (IOException e) { e.printStackTrace(); } + + // TODO (huydhn): Figure out on what the final JSON results looks like, we need something + // with the same number of fields as https://github.com/pytorch/pytorch/pull/135042 + try (FileWriter writer = new FileWriter(getFilesDir() + "/benchmark_results.json")) { + Gson gson = new Gson(); + writer.write(gson.toJson(stats)); + } catch (IOException e) { + e.printStackTrace(); + } + } +} + +class Stats { + List latency = new ArrayList<>(); + + @Override + public String toString() { + return "latency: " + latency.stream().map(Object::toString).collect(Collectors.joining("")); } } diff --git a/extension/android/benchmark/app/src/main/java/org/pytorch/minibench/LlmBenchmarkActivity.java b/extension/android/benchmark/app/src/main/java/org/pytorch/minibench/LlmBenchmarkActivity.java index 496cbde53d6..fc72b6affb0 100644 --- a/extension/android/benchmark/app/src/main/java/org/pytorch/minibench/LlmBenchmarkActivity.java +++ b/extension/android/benchmark/app/src/main/java/org/pytorch/minibench/LlmBenchmarkActivity.java @@ -22,7 +22,7 @@ public class LlmBenchmarkActivity extends Activity implements ModelRunnerCallbac ModelRunner mModelRunner; String mPrompt; - StatsInfo mStatsInfo; + LlmStats mLlmStats; @Override protected void onCreate(Bundle savedInstanceState) { @@ -44,20 +44,20 @@ protected void onCreate(Bundle savedInstanceState) { mPrompt = "The ultimate answer"; } - mStatsInfo = new StatsInfo(); + mLlmStats = new LlmStats(); mModelRunner = new ModelRunner(model.getPath(), tokenizerPath, temperature, this); - mStatsInfo.loadStart = System.currentTimeMillis(); + mLlmStats.loadStart = System.currentTimeMillis(); } @Override public void onModelLoaded(int status) { - mStatsInfo.loadEnd = System.currentTimeMillis(); + mLlmStats.loadEnd = System.currentTimeMillis(); if (status != 0) { Log.e("LlmBenchmarkRunner", "Loaded failed: " + status); onGenerationStopped(); return; } - mStatsInfo.generateStart = System.currentTimeMillis(); + mLlmStats.generateStart = System.currentTimeMillis(); mModelRunner.generate(mPrompt); } @@ -66,16 +66,16 @@ public void onTokenGenerated(String token) {} @Override public void onStats(String stats) { - mStatsInfo.tokens = stats; + mLlmStats.tokens = stats; } @Override public void onGenerationStopped() { - mStatsInfo.generateEnd = System.currentTimeMillis(); + mLlmStats.generateEnd = System.currentTimeMillis(); // TODO (huydhn): Remove txt files here once the JSON format is ready try (FileWriter writer = new FileWriter(getFilesDir() + "/benchmark_results.txt")) { - writer.write(mStatsInfo.toString()); + writer.write(mLlmStats.toString()); } catch (IOException e) { e.printStackTrace(); } @@ -84,14 +84,14 @@ public void onGenerationStopped() { // with the same number of fields as https://github.com/pytorch/pytorch/pull/135042 try (FileWriter writer = new FileWriter(getFilesDir() + "/benchmark_results.json")) { Gson gson = new Gson(); - writer.write(gson.toJson(mStatsInfo)); + writer.write(gson.toJson(mLlmStats)); } catch (IOException e) { e.printStackTrace(); } } } -class StatsInfo { +class LlmStats { long loadStart; long loadEnd; long generateStart; From e7003c5fb712512b89432bc3d8ad2783cb726451 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Wed, 11 Sep 2024 23:20:42 -0700 Subject: [PATCH 3/3] Llm undo change --- .../minibench/LlmBenchmarkActivity.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/extension/android/benchmark/app/src/main/java/org/pytorch/minibench/LlmBenchmarkActivity.java b/extension/android/benchmark/app/src/main/java/org/pytorch/minibench/LlmBenchmarkActivity.java index fc72b6affb0..496cbde53d6 100644 --- a/extension/android/benchmark/app/src/main/java/org/pytorch/minibench/LlmBenchmarkActivity.java +++ b/extension/android/benchmark/app/src/main/java/org/pytorch/minibench/LlmBenchmarkActivity.java @@ -22,7 +22,7 @@ public class LlmBenchmarkActivity extends Activity implements ModelRunnerCallbac ModelRunner mModelRunner; String mPrompt; - LlmStats mLlmStats; + StatsInfo mStatsInfo; @Override protected void onCreate(Bundle savedInstanceState) { @@ -44,20 +44,20 @@ protected void onCreate(Bundle savedInstanceState) { mPrompt = "The ultimate answer"; } - mLlmStats = new LlmStats(); + mStatsInfo = new StatsInfo(); mModelRunner = new ModelRunner(model.getPath(), tokenizerPath, temperature, this); - mLlmStats.loadStart = System.currentTimeMillis(); + mStatsInfo.loadStart = System.currentTimeMillis(); } @Override public void onModelLoaded(int status) { - mLlmStats.loadEnd = System.currentTimeMillis(); + mStatsInfo.loadEnd = System.currentTimeMillis(); if (status != 0) { Log.e("LlmBenchmarkRunner", "Loaded failed: " + status); onGenerationStopped(); return; } - mLlmStats.generateStart = System.currentTimeMillis(); + mStatsInfo.generateStart = System.currentTimeMillis(); mModelRunner.generate(mPrompt); } @@ -66,16 +66,16 @@ public void onTokenGenerated(String token) {} @Override public void onStats(String stats) { - mLlmStats.tokens = stats; + mStatsInfo.tokens = stats; } @Override public void onGenerationStopped() { - mLlmStats.generateEnd = System.currentTimeMillis(); + mStatsInfo.generateEnd = System.currentTimeMillis(); // TODO (huydhn): Remove txt files here once the JSON format is ready try (FileWriter writer = new FileWriter(getFilesDir() + "/benchmark_results.txt")) { - writer.write(mLlmStats.toString()); + writer.write(mStatsInfo.toString()); } catch (IOException e) { e.printStackTrace(); } @@ -84,14 +84,14 @@ public void onGenerationStopped() { // with the same number of fields as https://github.com/pytorch/pytorch/pull/135042 try (FileWriter writer = new FileWriter(getFilesDir() + "/benchmark_results.json")) { Gson gson = new Gson(); - writer.write(gson.toJson(mLlmStats)); + writer.write(gson.toJson(mStatsInfo)); } catch (IOException e) { e.printStackTrace(); } } } -class LlmStats { +class StatsInfo { long loadStart; long loadEnd; long generateStart;