Android App with MediaTek Mode

build/build_android_llm_demo.sh

@@ -37,6 +37,7 @@ build_android_native_library() {
   cmake . -DCMAKE_INSTALL_PREFIX="${CMAKE_OUT}" \
     -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK}/build/cmake/android.toolchain.cmake" \
     -DANDROID_ABI="${ANDROID_ABI}" \
+    -DANDROID_PLATFORM=android-26 \
     -DEXECUTORCH_ENABLE_LOGGING=ON \
     -DEXECUTORCH_LOG_LEVEL=Info \
     -DEXECUTORCH_BUILD_XNNPACK=ON \
@@ -65,6 +66,7 @@ build_android_native_library() {
   cmake extension/android \
     -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake \
     -DANDROID_ABI="${ANDROID_ABI}" \
+    -DANDROID_PLATFORM=android-26 \
     -DCMAKE_INSTALL_PREFIX="${CMAKE_OUT}" \
     -DEXECUTORCH_ENABLE_LOGGING=ON \
     -DEXECUTORCH_LOG_LEVEL=Info \

examples/demo-apps/android/LlamaDemo/app/src/main/AndroidManifest.xml

@@ -36,6 +36,30 @@
             android:name="libcdsprpc.so"
             android:required="false" />
 
+        <uses-native-library
+            android:name="libapuwareutils_v2.mtk.so"
+            android:required="false" />
+
+        <uses-native-library
+            android:name="libapuwareapusys_v2.mtk.so"
+            android:required="false" />
+
+        <uses-native-library
+            android:name="libnir_neon_driver_ndk.mtk.so"
+            android:required="false" />
+
+        <uses-native-library
+            android:name="libnir_neon_driver_ndk.mtk.vndk.so"
+            android:required="false" />
+
+        <uses-native-library
+            android:name="libcmdl_ndk.mtk.vndk.so"
+            android:required="false" />
+
+        <uses-native-library
+            android:name="libcmdl_ndk.mtk.so"
+            android:required="false" />
+
         <activity
             android:name=".MainActivity"
             android:exported="true"

examples/demo-apps/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/BackendType.java

@@ -0,0 +1,6 @@
+package com.example.executorchllamademo;
+
+public enum BackendType {
+  XNNPACK,
+  MEDIATEK
+}

examples/demo-apps/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/MainActivity.java

@@ -125,7 +125,8 @@ private void setLocalModel(String modelPath, String tokenizerPath, float tempera
     long runStartTime = System.currentTimeMillis();
     mModule =
         new LlamaModule(
-            ModelUtils.getModelCategory(mCurrentSettingsFields.getModelType()),
+            ModelUtils.getModelCategory(
+                mCurrentSettingsFields.getModelType(), mCurrentSettingsFields.getBackendType()),
             modelPath,
             tokenizerPath,
             temperature);
@@ -174,6 +175,11 @@ private void setLocalModel(String modelPath, String tokenizerPath, float tempera
             + modelPath
             + "\nTokenizer path: "
             + tokenizerPath
+            + "\nBackend: "
+            + mCurrentSettingsFields.getBackendType().toString()
+            + "\nModelType: "
+            + ModelUtils.getModelCategory(
+                mCurrentSettingsFields.getModelType(), mCurrentSettingsFields.getBackendType())
             + "\nTemperature: "
             + temperature
             + "\nModel loaded time: "
@@ -229,6 +235,7 @@ protected void onCreate(Bundle savedInstanceState) {
 
     try {
       Os.setenv("ADSP_LIBRARY_PATH", getApplicationInfo().nativeLibraryDir, true);
+      Os.setenv("LD_LIBRARY_PATH", getApplicationInfo().nativeLibraryDir, true);
     } catch (ErrnoException e) {
       finish();
     }
@@ -285,6 +292,7 @@ protected void onResume() {
       }
       boolean isUpdated = !mCurrentSettingsFields.equals(updatedSettingsFields);
       boolean isLoadModel = updatedSettingsFields.getIsLoadModel();
+      setBackendMode(updatedSettingsFields.getBackendType());
       if (isUpdated) {
         if (isLoadModel) {
           // If users change the model file, but not pressing loadModelButton, we won't load the new
@@ -293,6 +301,7 @@ protected void onResume() {
         } else {
           askUserToSelectModel();
         }
+
         checkForClearChatHistory(updatedSettingsFields);
         // Update current to point to the latest
         mCurrentSettingsFields = new SettingsFields(updatedSettingsFields);
@@ -302,6 +311,22 @@ protected void onResume() {
     }
   }
 
+  private void setBackendMode(BackendType backendType) {
+    if (backendType.equals(BackendType.XNNPACK)) {
+      setXNNPACKMode();
+    } else if (backendType.equals(BackendType.MEDIATEK)) {
+      setMediaTekMode();
+    }
+  }
+
+  private void setXNNPACKMode() {
+    requireViewById(R.id.addMediaButton).setVisibility(View.VISIBLE);
+  }
+
+  private void setMediaTekMode() {
+    requireViewById(R.id.addMediaButton).setVisibility(View.GONE);
+  }
+
   private void checkForClearChatHistory(SettingsFields updatedSettingsFields) {
     if (updatedSettingsFields.getIsClearChatHistory()) {
       mMessageAdapter.clear();
@@ -672,7 +697,8 @@ private void onModelRunStopped() {
           addSelectedImagesToChatThread(mSelectedImageUri);
           String finalPrompt;
           String rawPrompt = mEditTextMessage.getText().toString();
-          if (ModelUtils.getModelCategory(mCurrentSettingsFields.getModelType())
+          if (ModelUtils.getModelCategory(
+                  mCurrentSettingsFields.getModelType(), mCurrentSettingsFields.getBackendType())
               == ModelUtils.VISION_MODEL) {
             finalPrompt = mCurrentSettingsFields.getFormattedSystemAndUserPrompt(rawPrompt);
           } else {
@@ -705,7 +731,9 @@ public void run() {
                         }
                       });
                   long generateStartTime = System.currentTimeMillis();
-                  if (ModelUtils.getModelCategory(mCurrentSettingsFields.getModelType())
+                  if (ModelUtils.getModelCategory(
+                          mCurrentSettingsFields.getModelType(),
+                          mCurrentSettingsFields.getBackendType())
                       == ModelUtils.VISION_MODEL) {
                     mModule.generateFromPos(
                         finalPrompt,

examples/demo-apps/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/ModelUtils.java

@@ -9,21 +9,33 @@
 package com.example.executorchllamademo;
 
 public class ModelUtils {
+  // XNNPACK or QNN
   static final int TEXT_MODEL = 1;
+
+  // XNNPACK
   static final int VISION_MODEL = 2;
   static final int VISION_MODEL_IMAGE_CHANNELS = 3;
   static final int VISION_MODEL_SEQ_LEN = 768;
   static final int TEXT_MODEL_SEQ_LEN = 256;
 
-  public static int getModelCategory(ModelType modelType) {
-    switch (modelType) {
-      case LLAVA_1_5:
-        return VISION_MODEL;
-      case LLAMA_3:
-      case LLAMA_3_1:
-      case LLAMA_3_2:
-      default:
-        return TEXT_MODEL;
+  // MediaTek
+  static final int MEDIATEK_TEXT_MODEL = 3;
+
+  public static int getModelCategory(ModelType modelType, BackendType backendType) {
+    if (backendType.equals(BackendType.XNNPACK)) {
+      switch (modelType) {
+        case LLAVA_1_5:
+          return VISION_MODEL;
+        case LLAMA_3:
+        case LLAMA_3_1:
+        case LLAMA_3_2:
+        default:
+          return TEXT_MODEL;
+      }
+    } else if (backendType.equals(BackendType.MEDIATEK)) {
+      return MEDIATEK_TEXT_MODEL;
     }
+
+    return TEXT_MODEL; // default
   }
 }

examples/demo-apps/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/SettingsActivity.java

@@ -14,6 +14,7 @@
 import android.os.Bundle;
 import android.text.Editable;
 import android.text.TextWatcher;
+import android.view.View;
 import android.widget.Button;
 import android.widget.EditText;
 import android.widget.ImageButton;
@@ -32,6 +33,7 @@ public class SettingsActivity extends AppCompatActivity {
 
   private String mModelFilePath = "";
   private String mTokenizerFilePath = "";
+  private TextView mBackendTextView;
   private TextView mModelTextView;
   private TextView mTokenizerTextView;
   private TextView mModelTypeTextView;
@@ -41,6 +43,7 @@ public class SettingsActivity extends AppCompatActivity {
   private double mSetTemperature;
   private String mSystemPrompt;
   private String mUserPrompt;
+  private BackendType mBackendType;
   private ModelType mModelType;
   public SettingsFields mSettingsFields;
 
@@ -68,9 +71,11 @@ protected void onCreate(Bundle savedInstanceState) {
   }
 
   private void setupSettings() {
+    mBackendTextView = requireViewById(R.id.backendTextView);
     mModelTextView = requireViewById(R.id.modelTextView);
     mTokenizerTextView = requireViewById(R.id.tokenizerTextView);
     mModelTypeTextView = requireViewById(R.id.modelTypeTextView);
+    ImageButton backendImageButton = requireViewById(R.id.backendImageButton);
     ImageButton modelImageButton = requireViewById(R.id.modelImageButton);
     ImageButton tokenizerImageButton = requireViewById(R.id.tokenizerImageButton);
     ImageButton modelTypeImageButton = requireViewById(R.id.modelTypeImageButton);
@@ -79,6 +84,10 @@ private void setupSettings() {
     loadSettings();
 
     // TODO: The two setOnClickListeners will be removed after file path issue is resolved
+    backendImageButton.setOnClickListener(
+        view -> {
+          setupBackendSelectorDialog();
+        });
     modelImageButton.setOnClickListener(
         view -> {
           setupModelSelectorDialog();
@@ -104,6 +113,12 @@ private void setupSettings() {
     if (mModelType != null) {
       mModelTypeTextView.setText(mModelType.toString());
     }
+    mBackendType = mSettingsFields.getBackendType();
+    ETLogging.getInstance().log("mBackendType from settings " + mBackendType);
+    if (mBackendType != null) {
+      mBackendTextView.setText(mBackendType.toString());
+      setBackendSettingMode();
+    }
 
     setupParameterSettings();
     setupPromptSettings();
@@ -285,6 +300,29 @@ private void showInvalidPromptDialog() {
         .show();
   }
 
+  private void setupBackendSelectorDialog() {
+    // Convert enum to list
+    List<String> backendTypesList = new ArrayList<>();
+    for (BackendType backendType : BackendType.values()) {
+      backendTypesList.add(backendType.toString());
+    }
+    // Alert dialog builder takes in arr of string instead of list
+    String[] backendTypes = backendTypesList.toArray(new String[0]);
+    AlertDialog.Builder backendTypeBuilder = new AlertDialog.Builder(this);
+    backendTypeBuilder.setTitle("Select backend type");
+    backendTypeBuilder.setSingleChoiceItems(
+        backendTypes,
+        -1,
+        (dialog, item) -> {
+          mBackendTextView.setText(backendTypes[item]);
+          mBackendType = BackendType.valueOf(backendTypes[item]);
+          setBackendSettingMode();
+          dialog.dismiss();
+        });
+
+    backendTypeBuilder.create().show();
+  }
+
   private void setupModelSelectorDialog() {
     String[] pteFiles = listLocalFile("/data/local/tmp/llama/", ".pte");
     AlertDialog.Builder modelPathBuilder = new AlertDialog.Builder(this);
@@ -370,6 +408,32 @@ private String getFilenameFromPath(String uriFilePath) {
     return "";
   }
 
+  private void setBackendSettingMode() {
+    if (mBackendType.equals(BackendType.XNNPACK)) {
+      setXNNPACKSettingMode();
+    } else if (mBackendType.equals(BackendType.MEDIATEK)) {
+      setMediaTekSettingMode();
+    }
+  }
+
+  private void setXNNPACKSettingMode() {
+    requireViewById(R.id.modelLayout).setVisibility(View.VISIBLE);
+    requireViewById(R.id.tokenizerLayout).setVisibility(View.VISIBLE);
+    requireViewById(R.id.parametersView).setVisibility(View.VISIBLE);
+    requireViewById(R.id.temperatureLayout).setVisibility(View.VISIBLE);
+    mModelFilePath = "";
+    mTokenizerFilePath = "";
+  }
+
+  private void setMediaTekSettingMode() {
+    requireViewById(R.id.modelLayout).setVisibility(View.GONE);
+    requireViewById(R.id.tokenizerLayout).setVisibility(View.GONE);
+    requireViewById(R.id.parametersView).setVisibility(View.GONE);
+    requireViewById(R.id.temperatureLayout).setVisibility(View.GONE);
+    mModelFilePath = "/in/mtk/llama/runner";
+    mTokenizerFilePath = "/in/mtk/llama/runner";
+  }
+
   private void loadSettings() {
     Gson gson = new Gson();
     String settingsFieldsJSON = mDemoSharedPreferences.getSettings();
@@ -384,6 +448,7 @@ private void saveSettings() {
     mSettingsFields.saveParameters(mSetTemperature);
     mSettingsFields.savePrompts(mSystemPrompt, mUserPrompt);
     mSettingsFields.saveModelType(mModelType);
+    mSettingsFields.saveBackendType(mBackendType);
     mDemoSharedPreferences.addSettings(mSettingsFields);
   }
 

examples/demo-apps/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/SettingsFields.java

@@ -30,6 +30,10 @@ public ModelType getModelType() {
     return modelType;
   }
 
+  public BackendType getBackendType() {
+    return backendType;
+  }
+
   public String getUserPrompt() {
     return userPrompt;
   }
@@ -63,9 +67,11 @@ public boolean getIsLoadModel() {
   private boolean isClearChatHistory;
   private boolean isLoadModel;
   private ModelType modelType;
+  private BackendType backendType;
 
   public SettingsFields() {
     ModelType DEFAULT_MODEL = ModelType.LLAMA_3;
+    BackendType DEFAULT_BACKEND = BackendType.XNNPACK;
 
     modelFilePath = "";
     tokenizerFilePath = "";
@@ -75,6 +81,7 @@ public SettingsFields() {
     isClearChatHistory = false;
     isLoadModel = false;
     modelType = DEFAULT_MODEL;
+    backendType = DEFAULT_BACKEND;
   }
 
   public SettingsFields(SettingsFields settingsFields) {
@@ -86,6 +93,7 @@ public SettingsFields(SettingsFields settingsFields) {
     this.isClearChatHistory = settingsFields.getIsClearChatHistory();
     this.isLoadModel = settingsFields.getIsLoadModel();
     this.modelType = settingsFields.modelType;
+    this.backendType = settingsFields.backendType;
   }
 
   public void saveModelPath(String modelFilePath) {
@@ -100,6 +108,10 @@ public void saveModelType(ModelType modelType) {
     this.modelType = modelType;
   }
 
+  public void saveBackendType(BackendType backendType) {
+    this.backendType = backendType;
+  }
+
   public void saveParameters(Double temperature) {
     this.temperature = temperature;
   }
@@ -126,6 +138,7 @@ public boolean equals(SettingsFields anotherSettingsFields) {
         && userPrompt.equals(anotherSettingsFields.userPrompt)
         && isClearChatHistory == anotherSettingsFields.isClearChatHistory
         && isLoadModel == anotherSettingsFields.isLoadModel
-        && modelType == anotherSettingsFields.modelType;
+        && modelType == anotherSettingsFields.modelType
+        && backendType == anotherSettingsFields.backendType;
   }
 }