core: add more options for chat settings #38

shubham0204 · shubham0204 · commit c71ee400ad0f · 2025-02-22T19:26:48.000+05:30
add options to configure chat-template, use_mmap, use_mlock, n_threads
add option to clear all chat messages
diff --git a/.clang-format b/.clang-format
@@ -0,0 +1,21 @@
+---
+BasedOnStyle: GNU
+Language: Cpp
+IndentWidth: 4
+UseTab: Never
+PointerAlignment: Left
+ReferenceAlignment: Left
+AlwaysBreakAfterReturnType: None
+SpaceBeforeParens: ControlStatements
+IndentPPDirectives: None
+BreakBeforeBraces: Attach
+ColumnLimit: 120
+AlignConsecutiveDeclarations: true
+AlignConsecutiveAssignments: true
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+IndentCaseLabels: false
+KeepEmptyLinesAtTheStartOfBlocks: false
+MaxEmptyLinesToKeep: 1
+SortIncludes: true
+IndentWrappedFunctionNames: false
diff --git a/.gitignore b/.gitignore
@@ -9,3 +9,5 @@
 .cxx
 local.properties
 .kotlin
+ktlint
+ktlint.bat
diff --git a/app/objectbox-models/default.json b/app/objectbox-models/default.json
@@ -5,7 +5,7 @@
   "entities": [
     {
       "id": "1:2836016800019494484",
-      "lastPropertyId": "11:2362849161259116908",
+      "lastPropertyId": "15:5655950154838757763",
       "name": "Chat",
       "properties": [
         {
@@ -63,6 +63,26 @@
           "id": "11:2362849161259116908",
           "name": "contextSizeConsumed",
           "type": 5
+        },
+        {
+          "id": "12:454958936887824250",
+          "name": "chatTemplate",
+          "type": 9
+        },
+        {
+          "id": "13:4856054665752880536",
+          "name": "nThreads",
+          "type": 5
+        },
+        {
+          "id": "14:3913930867701003089",
+          "name": "useMmap",
+          "type": 1
+        },
+        {
+          "id": "15:5655950154838757763",
+          "name": "useMlock",
+          "type": 1
         }
       ],
       "relations": []
@@ -98,7 +118,7 @@
     },
     {
       "id": "3:3976296884919346124",
-      "lastPropertyId": "5:5106603935737186708",
+      "lastPropertyId": "6:9128691076308457471",
       "name": "LLMModel",
       "properties": [
         {
@@ -126,6 +146,11 @@
           "id": "5:5106603935737186708",
           "name": "contextSize",
           "type": 5
+        },
+        {
+          "id": "6:9128691076308457471",
+          "name": "chatTemplate",
+          "type": 9
         }
       ],
       "relations": []
diff --git a/app/src/main/java/io/shubham0204/smollmandroid/data/ChatsDB.kt b/app/src/main/java/io/shubham0204/smollmandroid/data/ChatsDB.kt
@@ -43,6 +43,10 @@ data class Chat(
     var isTask: Boolean = false,
     var contextSize: Int = 0,
     var contextSizeConsumed: Int = 0,
+    var chatTemplate: String = "",
+    var nThreads: Int = 4,
+    var useMmap: Boolean = true,
+    var useMlock: Boolean = false,
 )
 
 @Single
diff --git a/app/src/main/java/io/shubham0204/smollmandroid/data/ModelsDB.kt b/app/src/main/java/io/shubham0204/smollmandroid/data/ModelsDB.kt
@@ -31,6 +31,7 @@ data class LLMModel(
     var url: String = "",
     var path: String = "",
     var contextSize: Int = 0,
+    var chatTemplate: String = "",
 )
 
 @Single
@@ -42,11 +43,25 @@ class ModelsDB {
         url: String,
         path: String,
         contextSize: Int,
+        chatTemplate: String,
     ) {
-        modelsBox.put(LLMModel(name = name, url = url, path = path, contextSize = contextSize))
+        modelsBox.put(
+            LLMModel(
+                name = name,
+                url = url,
+                path = path,
+                contextSize = contextSize,
+                chatTemplate = chatTemplate,
+            ),
+        )
     }
 
-    fun getModel(id: Long): LLMModel? = modelsBox.get(id)
+    fun getModel(id: Long): LLMModel? =
+        try {
+            modelsBox.get(id)
+        } catch (e: IllegalArgumentException) {
+            null
+        }
 
     fun getModels(): Flow<List<LLMModel>> =
         modelsBox
diff --git a/app/src/main/java/io/shubham0204/smollmandroid/llm/SmolLMManager.kt b/app/src/main/java/io/shubham0204/smollmandroid/llm/SmolLMManager.kt
@@ -40,7 +40,7 @@ class SmolLMManager(
     private var responseGenerationJob: Job? = null
     private var modelInitJob: Job? = null
     private var chat: Chat? = null
-    var isInstanceLoaded = false
+    private var isInstanceLoaded = false
     var isInferenceOn = false
 
     data class SmolLMInitParams(
@@ -50,6 +50,10 @@ class SmolLMManager(
         val temperature: Float,
         val storeChats: Boolean,
         val contextSize: Long,
+        val chatTemplate: String,
+        val nThreads: Int,
+        val useMmap: Boolean,
+        val useMlock: Boolean,
     )
 
     data class SmolLMResponse(
@@ -77,6 +81,10 @@ class SmolLMManager(
                         initParams.temperature,
                         initParams.storeChats,
                         initParams.contextSize,
+                        initParams.chatTemplate,
+                        initParams.nThreads,
+                        initParams.useMmap,
+                        initParams.useMlock,
                     )
                     LOGD("Model loaded")
                     if (initParams.chat.systemPrompt.isNotEmpty()) {
diff --git a/app/src/main/java/io/shubham0204/smollmandroid/ui/screens/chat/ChatActivity.kt b/app/src/main/java/io/shubham0204/smollmandroid/ui/screens/chat/ChatActivity.kt
@@ -155,9 +155,6 @@ class ChatActivity : ComponentActivity() {
     override fun onStop() {
         super.onStop()
         modelUnloaded = viewModel.unloadModel()
-        if (modelUnloaded) {
-            Toast.makeText(this, "Model unloaded from memory", Toast.LENGTH_SHORT).show()
-        }
         LOGD("onStop() called - model unloaded result: $modelUnloaded")
     }
 }
@@ -644,7 +641,7 @@ private fun SelectModelsList(viewModel: ChatScreenViewModel) {
             onDismissRequest = { viewModel.hideSelectModelListDialog() },
             modelsList,
             onModelListItemClick = { model ->
-                viewModel.updateChatLLM(model.id)
+                viewModel.updateChatLLMParams(model.id, model.chatTemplate)
                 viewModel.loadModel()
                 viewModel.hideSelectModelListDialog()
             },
diff --git a/app/src/main/java/io/shubham0204/smollmandroid/ui/screens/chat/ChatMoreOptionsPopup.kt b/app/src/main/java/io/shubham0204/smollmandroid/ui/screens/chat/ChatMoreOptionsPopup.kt
@@ -94,5 +94,30 @@ fun ChatMoreOptionsPopup(
                 viewModel.hideMoreOptionsPopup()
             },
         )
+        DropdownMenuItem(
+            leadingIcon = { Icon(Icons.Default.Delete, contentDescription = "Clear Chat Messages") },
+            text = { Text(stringResource(R.string.chat_options_clear_messages)) },
+            onClick = {
+                viewModel.currChatState.value?.let { chat ->
+                    createAlertDialog(
+                        dialogTitle = context.getString(R.string.chat_options_clear_messages),
+                        dialogText = context.getString(R.string.chat_options_clear_messages_text),
+                        dialogPositiveButtonText = context.getString(R.string.dialog_pos_clear),
+                        dialogNegativeButtonText = context.getString(R.string.dialog_neg_cancel),
+                        onPositiveButtonClick = {
+                            viewModel.deleteChatMessages(chat)
+                            Toast
+                                .makeText(
+                                    viewModel.context,
+                                    "Chat '${chat.name}' cleared",
+                                    Toast.LENGTH_LONG,
+                                ).show()
+                        },
+                        onNegativeButtonClick = {},
+                    )
+                }
+                viewModel.hideMoreOptionsPopup()
+            },
+        )
     }
 }
diff --git a/app/src/main/java/io/shubham0204/smollmandroid/ui/screens/chat/ChatScreenViewModel.kt b/app/src/main/java/io/shubham0204/smollmandroid/ui/screens/chat/ChatScreenViewModel.kt
@@ -147,8 +147,11 @@ class ChatScreenViewModel(
 
     fun getChatMessages(chatId: Long): Flow<List<ChatMessage>> = messagesDB.getMessages(chatId)
 
-    fun updateChatLLM(modelId: Long) {
-        _currChatState.value = _currChatState.value?.copy(llmModelId = modelId)
+    fun updateChatLLMParams(
+        modelId: Long,
+        chatTemplate: String,
+    ) {
+        _currChatState.value = _currChatState.value?.copy(llmModelId = modelId, chatTemplate = chatTemplate)
         chatsDB.updateChat(_currChatState.value!!)
     }
 
@@ -223,6 +226,11 @@ class ChatScreenViewModel(
         _currChatState.value = null
     }
 
+    fun deleteChatMessages(chat: Chat) {
+        stopGeneration()
+        messagesDB.deleteMessages(chat.id)
+    }
+
     fun deleteModel(modelId: Long) {
         modelsRepository.deleteModel(modelId)
         if (_currChatState.value?.llmModelId == modelId) {
@@ -252,6 +260,10 @@ class ChatScreenViewModel(
                         chat.temperature,
                         !chat.isTask,
                         chat.contextSize.toLong(),
+                        chat.chatTemplate,
+                        chat.nThreads,
+                        chat.useMmap,
+                        chat.useMlock,
                     ),
                     onError = { e ->
                         _modelLoadState.value = ModelLoadingState.FAILURE
diff --git a/app/src/main/java/io/shubham0204/smollmandroid/ui/screens/chat/EditChatSettingsScreen.kt b/app/src/main/java/io/shubham0204/smollmandroid/ui/screens/chat/EditChatSettingsScreen.kt
@@ -73,9 +73,16 @@ fun EditChatSettingsScreen(
         var minP by remember { mutableFloatStateOf(chat.minP) }
         var temperature by remember { mutableFloatStateOf(chat.temperature) }
         var contextSize by remember { mutableIntStateOf(chat.contextSize) }
+        var nThreads by remember { mutableIntStateOf(chat.nThreads) }
         var takeContextSizeFromModel by remember { mutableStateOf(false) }
+        var chatTemplate by remember { mutableStateOf(chat.chatTemplate) }
+        var useMmap by remember { mutableStateOf(chat.useMmap) }
+        var useMlock by remember { mutableStateOf(chat.useMlock) }
         val context = LocalContext.current
         val llmModel = viewModel.modelsRepository.getModelFromId(chat.llmModelId)
+
+        val totalThreads = Runtime.getRuntime().availableProcessors()
+
         SmolLMAndroidTheme {
             Scaffold(
                 topBar = {
@@ -99,6 +106,10 @@ fun EditChatSettingsScreen(
                                             minP = minP,
                                             temperature = temperature,
                                             contextSize = contextSize,
+                                            chatTemplate = chatTemplate,
+                                            nThreads = nThreads,
+                                            useMmap = useMmap,
+                                            useMlock = useMlock,
                                         )
                                     if (chat != updatedChat) {
                                         viewModel.updateChat(updatedChat)
@@ -152,6 +163,21 @@ fun EditChatSettingsScreen(
                             KeyboardOptions.Default.copy(
                                 capitalization = KeyboardCapitalization.Sentences,
                             ),
+                        maxLines = 5,
+                    )
+
+                    Spacer(modifier = Modifier.height(16.dp))
+
+                    TextField(
+                        modifier = Modifier.fillMaxWidth(),
+                        value = chatTemplate,
+                        onValueChange = { chatTemplate = it },
+                        label = { Text("Chat template") },
+                        keyboardOptions =
+                            KeyboardOptions.Default.copy(
+                                capitalization = KeyboardCapitalization.Sentences,
+                            ),
+                        maxLines = 5,
                     )
 
                     Spacer(modifier = Modifier.height(8.dp))
@@ -266,6 +292,64 @@ fun EditChatSettingsScreen(
                             style = MaterialTheme.typography.labelSmall,
                         )
                     }
+                    Spacer(modifier = Modifier.height(24.dp))
+
+                    Text(
+                        text = "Number of Threads",
+                        style = MaterialTheme.typography.titleMedium,
+                    )
+                    Text(
+                        text = "num threads desc",
+                        style = MaterialTheme.typography.labelSmall,
+                    )
+                    Slider(
+                        value = nThreads.toFloat(),
+                        onValueChange = { nThreads = it.toInt() },
+                        valueRange = 0.0f..(totalThreads).toFloat(),
+                        steps = totalThreads,
+                    )
+                    Text(
+                        text = "$nThreads",
+                        style = MaterialTheme.typography.labelSmall,
+                    )
+
+                    Spacer(modifier = Modifier.height(24.dp))
+
+                    Row(modifier = Modifier.fillMaxWidth()) {
+                        Checkbox(
+                            checked = useMmap,
+                            onCheckedChange = { useMmap = it },
+                        )
+                        Column {
+                            Text(
+                                text = "Use mmap",
+                                style = MaterialTheme.typography.titleMedium,
+                            )
+                            Text(
+                                text = "Disable memory mapping (mmap) for potentially fewer pageouts and better performance on low-memory systems, but with slower load times and a risk of preventing loading large models.",
+                                style = MaterialTheme.typography.labelSmall,
+                            )
+                        }
+                    }
+                    Spacer(modifier = Modifier.height(24.dp))
+
+                    Row(modifier = Modifier.fillMaxWidth()) {
+                        Checkbox(
+                            checked = useMlock,
+                            onCheckedChange = { useMlock = it },
+                        )
+                        Column {
+                            Text(
+                                text = "Use mlock",
+                                style = MaterialTheme.typography.titleMedium,
+                            )
+                            Text(
+                                text = "Keep the model loaded in RAM for faster performance, but uses more memory and may load slower.",
+                                style = MaterialTheme.typography.labelSmall,
+                            )
+                        }
+                    }
+                    Spacer(modifier = Modifier.height(24.dp))
                 }
             }
         }
diff --git a/app/src/main/java/io/shubham0204/smollmandroid/ui/screens/model_download/DownloadModelsViewModel.kt b/app/src/main/java/io/shubham0204/smollmandroid/ui/screens/model_download/DownloadModelsViewModel.kt
@@ -122,11 +122,14 @@ class DownloadModelsViewModel(
                 val ggufReader = GGUFReader()
                 ggufReader.load(File(context.filesDir, fileName).absolutePath)
                 val contextSize = ggufReader.getContextSize() ?: -1
+                // TODO: Add a default chat template when the model does not contain one
+                val chatTemplate = ggufReader.getChatTemplate() ?: ""
                 modelsDB.addModel(
                     fileName,
                     "",
                     Paths.get(context.filesDir.absolutePath, fileName).toString(),
                     contextSize.toInt(),
+                    chatTemplate,
                 )
                 withContext(Dispatchers.Main) {
                     hideProgressDialog()
diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml
@@ -79,4 +79,7 @@
     <string name="dialog_progress_copy_model_title">Copying model file...</string>
     <string name="dialog_progress_copy_model_text">%1$s\n(The model needs to be copied to the app\'s internal directory for use)</string>
     <string name="toast_invalid_file">Invalid file</string>
+    <string name="chat_options_clear_messages">Clear Chat Messages</string>
+    <string name="chat_options_clear_messages_text">Are you sure you clear all messages from this chat?</string>
+    <string name="dialog_pos_clear">Clear</string>
 </resources>
diff --git a/smollm/src/main/cpp/GGUFReader.cpp b/smollm/src/main/cpp/GGUFReader.cpp
diff --git a/smollm/src/main/cpp/LLMInference.cpp b/smollm/src/main/cpp/LLMInference.cpp
diff --git a/smollm/src/main/cpp/LLMInference.h b/smollm/src/main/cpp/LLMInference.h
diff --git a/smollm/src/main/cpp/smollm.cpp b/smollm/src/main/cpp/smollm.cpp
diff --git a/smollm/src/main/java/io/shubham0204/smollm/GGUFReader.kt b/smollm/src/main/java/io/shubham0204/smollm/GGUFReader.kt
diff --git a/smollm/src/main/java/io/shubham0204/smollm/SmolLM.kt b/smollm/src/main/java/io/shubham0204/smollm/SmolLM.kt
