Skip to content

Commit a153de9

Browse files
songgotsonggot
committed
Add On Device LXM Service API and LLM example
- Add On Device LXM Service API - Add LLM example Signed-off-by: hyunil park <[email protected]>
1 parent 0c5c140 commit a153de9

File tree

9 files changed

+856
-0
lines changed

9 files changed

+856
-0
lines changed

‎Tizen.platform/lxm_service/README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
title: On device Machine LXM(LLM, LVM, etc.) Service API example
3+
...
4+
5+
## On device Machine LXM(LLM, LVM, etc.) Service API example

‎Tizen.platform/lxm_service/config.conf‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"single" :
3+
{
4+
"framework" : "flare",
5+
"model" : ["sflare_if_4bit_3b.bin"],
6+
"adapter" : ["history_lora.bin"],
7+
"custom" : "tokenizer_path:tokenizer.json,backend:CPU,output_size:1024,model_type:3B,data_type:W4A32",
8+
"invoke_dynamic" : "true",
9+
"invoke_async" : "false"
10+
}
11+
}

‎Tizen.platform/lxm_service/example/main.cc‎

Lines changed: 336 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,336 @@
1+
/**
2+
* @file main.cc
3+
* @date 23 JULY 2025
4+
* @brief Large Language Model service example using ML LXM Service API
5+
* @see https://github.com/nnstreamer/nnstreamer
6+
* @author <[email protected]>
7+
* @bug No known bugs.
8+
*/
9+
#include <glib.h>
10+
#include <iostream>
11+
#include <stdio.h>
12+
#include <stdlib.h>
13+
#include <unistd.h>
14+
#include <ml-api-service.h>
15+
#include "ml-lxm-service.h"
16+
17+
#define MAX_STRING_LEN 4096
18+
19+
enum {
20+
CURRENT_STATUS_MAINMENU,
21+
CURRENT_STATUS_SESSION_CREATE,
22+
CURRENT_STATUS_SESSION_DESTROY,
23+
CURRENT_STATUS_SESSION_SET_INSTRUCTION,
24+
CURRENT_STATUS_SESSION_RESPOND,
25+
CURRENT_STATUS_SESSION_RESPOND_ASYNC,
26+
CURRENT_STATUS_PROMPT_CREATE,
27+
CURRENT_STATUS_PROMPT_APPEND_TEXT,
28+
CURRENT_STATUS_PROMPT_APPEND_INSTRUCTION,
29+
CURRENT_STATUS_PROMPT_DESTROY
30+
};
31+
32+
int g_menu_state = CURRENT_STATUS_MAINMENU;
33+
GMainLoop *loop;
34+
ml_lxm_session_h g_session = NULL; // Global session handle
35+
ml_lxm_prompt_h g_prompt = NULL; // Global prompt handle
36+
char g_instructions[MAX_STRING_LEN]; // Buffer for instructions
37+
char g_prompt_text[MAX_STRING_LEN]; // Buffer for prompt text
38+
39+
/**
40+
* @brief Token callback for async response
41+
*/
42+
static void token_handler(ml_service_event_e event, ml_information_h event_data,
43+
void *user_data) {
44+
ml_tensors_data_h data = NULL;
45+
void *_raw = NULL;
46+
size_t _size = 0;
47+
int ret;
48+
49+
switch (event) {
50+
case ML_SERVICE_EVENT_NEW_DATA:
51+
if (event_data != NULL) {
52+
53+
ret = ml_information_get(event_data, "data", &data);
54+
if (ret != ML_ERROR_NONE) {
55+
g_print("Failed to get data from event_data\n");
56+
return;
57+
}
58+
59+
ret = ml_tensors_data_get_tensor_data(data, 0U, &_raw, &_size);
60+
if (ret != ML_ERROR_NONE) {
61+
g_print("Failed to get tensor data\n");
62+
return;
63+
}
64+
65+
std::cout.write(static_cast<const char *>(_raw),
66+
_size); /* korean output */
67+
std::cout.flush();
68+
}
69+
default:
70+
break;
71+
}
72+
}
73+
74+
/**
75+
* @brief Main menu display
76+
*/
77+
void main_menu() {
78+
g_print("\n");
79+
g_print("================================================================\n");
80+
g_print(" ML LLM Service Test (press q to quit) \n");
81+
g_print("----------------------------------------------------------------\n");
82+
g_print("a. Session Create \n");
83+
g_print("b. Session Destroy \n");
84+
g_print("c. Session Set Instruction \n");
85+
g_print("d. Session Respond \n");
86+
g_print("e. Prompt Create \n");
87+
g_print("f. Prompt Destroy \n");
88+
g_print("g. Prompt Append Text \n");
89+
g_print("h. Prompt Append Instruction \n");
90+
g_print("q. Quit \n");
91+
g_print("================================================================\n");
92+
}
93+
94+
/**
95+
* @brief Quit program
96+
*/
97+
static void quit_program() {
98+
// Cleanup before exit
99+
if (g_prompt) {
100+
ml_lxm_prompt_destroy(g_prompt);
101+
g_prompt = NULL;
102+
}
103+
if (g_session) {
104+
ml_lxm_session_destroy(g_session);
105+
g_session = NULL;
106+
}
107+
g_main_loop_quit(loop);
108+
}
109+
110+
/**
111+
* @brief Reset menu status
112+
*/
113+
void reset_menu_status(void) { g_menu_state = CURRENT_STATUS_MAINMENU; }
114+
115+
/**
116+
* @brief Display current menu
117+
*/
118+
static void display_menu() {
119+
if (g_menu_state == CURRENT_STATUS_MAINMENU) {
120+
main_menu();
121+
} else if (g_menu_state == CURRENT_STATUS_SESSION_CREATE) {
122+
g_print("*** Enter config path: ");
123+
} else if (g_menu_state == CURRENT_STATUS_SESSION_SET_INSTRUCTION) {
124+
g_print("*** Enter new instructions: ");
125+
} else if (g_menu_state == CURRENT_STATUS_PROMPT_APPEND_TEXT) {
126+
g_print("*** Enter text to append: ");
127+
} else if (g_menu_state == CURRENT_STATUS_PROMPT_APPEND_INSTRUCTION) {
128+
g_print("*** Enter instruction to append: ");
129+
} else {
130+
g_print("*** Unknown status. \n");
131+
reset_menu_status();
132+
}
133+
}
134+
135+
/**
136+
* @brief Interpret main menu commands
137+
*/
138+
static void interpret_main_menu(char cmd) {
139+
int ret = ML_ERROR_NONE;
140+
141+
ml_lxm_generation_options_s options = {.temperature = 1.2, .max_tokens = 128};
142+
143+
switch (cmd) {
144+
case 'a':
145+
if (g_session) {
146+
g_print("Session already exists!\n");
147+
break;
148+
}
149+
g_menu_state = CURRENT_STATUS_SESSION_CREATE;
150+
break;
151+
case 'b':
152+
if (!g_session) {
153+
g_print("No active session!\n");
154+
break;
155+
}
156+
ret = ml_lxm_session_destroy(g_session);
157+
if (ret == 0) {
158+
g_session = NULL;
159+
g_print("Session destroyed.\n");
160+
} else {
161+
g_print("Session destruction failed: %d\n", ret);
162+
}
163+
break;
164+
case 'c':
165+
if (!g_session) {
166+
g_print("Create a session first!\n");
167+
break;
168+
}
169+
g_menu_state = CURRENT_STATUS_SESSION_SET_INSTRUCTION;
170+
break;
171+
172+
case 'd':
173+
if (!g_session || !g_prompt) {
174+
g_print("Create session and prompt first!\n");
175+
break;
176+
}
177+
178+
ret = ml_lxm_session_respond(g_session, g_prompt, &options,
179+
token_handler, NULL);
180+
if (ret == 0) {
181+
g_print("\nAsync response started...\n");
182+
} else {
183+
g_print("Async response failed: %d\n", ret);
184+
}
185+
break;
186+
case 'e':
187+
if (g_prompt) {
188+
g_print("Prompt already exists! Destroy first.\n");
189+
break;
190+
}
191+
ret = ml_lxm_prompt_create(&g_prompt);
192+
if (ret == 0) {
193+
g_print("Prompt created.\n");
194+
} else {
195+
g_print("Prompt creation failed: %d\n", ret);
196+
}
197+
break;
198+
case 'f':
199+
if (!g_prompt) {
200+
g_print("No active prompt!\n");
201+
break;
202+
}
203+
ret = ml_lxm_prompt_destroy(g_prompt);
204+
if (ret == 0) {
205+
g_prompt = NULL;
206+
g_print("Prompt destroyed.\n");
207+
} else {
208+
g_print("Prompt destruction failed: %d\n", ret);
209+
}
210+
break;
211+
case 'g':
212+
if (!g_prompt) {
213+
g_print("Create a prompt first!\n");
214+
break;
215+
}
216+
g_menu_state = CURRENT_STATUS_PROMPT_APPEND_TEXT;
217+
break;
218+
219+
case 'h':
220+
if (!g_prompt) {
221+
g_print("Create a prompt first!\n");
222+
break;
223+
}
224+
g_menu_state = CURRENT_STATUS_PROMPT_APPEND_INSTRUCTION;
225+
break;
226+
case 'q':
227+
quit_program();
228+
return;
229+
default:
230+
g_print("Invalid option!\n");
231+
}
232+
233+
if (ret != ML_ERROR_NONE && cmd != 'q') {
234+
g_print("Operation failed with error: %d\n", ret);
235+
}
236+
}
237+
238+
/**
239+
* @brief Handle input for non-main-menu states
240+
*/
241+
static void handle_special_input(const char *input) {
242+
int ret = ML_ERROR_NONE;
243+
244+
switch (g_menu_state) {
245+
case CURRENT_STATUS_SESSION_CREATE:
246+
g_critical("Input: %s\n", input);
247+
ret = ml_lxm_session_create(&g_session, input, "Default instructions.");
248+
if (ret == 0) {
249+
g_print("Session created successfully.\n");
250+
} else {
251+
g_print("Session creation failed: %d\n", ret);
252+
}
253+
break;
254+
case CURRENT_STATUS_SESSION_SET_INSTRUCTION:
255+
ret = ml_lxm_session_set_instructions(g_session, input);
256+
if (ret == 0) {
257+
snprintf(g_instructions, MAX_STRING_LEN, "%s", input);
258+
g_print("Instructions updated.\n");
259+
} else {
260+
g_print("Failed to set instructions: %d\n", ret);
261+
}
262+
break;
263+
case CURRENT_STATUS_PROMPT_APPEND_TEXT:
264+
ret = ml_lxm_prompt_append_text(g_prompt, input);
265+
if (ret == 0) {
266+
snprintf(g_prompt_text, MAX_STRING_LEN, "%s", input);
267+
g_print("Text appended to prompt.\n");
268+
} else {
269+
g_print("Failed to append text: %d\n", ret);
270+
}
271+
break;
272+
case CURRENT_STATUS_PROMPT_APPEND_INSTRUCTION:
273+
ret = ml_lxm_prompt_append_instruction(g_prompt, input);
274+
if (ret == 0) {
275+
snprintf(g_prompt_text, MAX_STRING_LEN, "%s", input);
276+
g_print("Instruction appended to prompt.\n");
277+
} else {
278+
g_print("Failed to append instruction: %d\n", ret);
279+
}
280+
break;
281+
282+
default:
283+
break;
284+
}
285+
286+
reset_menu_status();
287+
}
288+
289+
/**
290+
* @brief Input handler
291+
*/
292+
gboolean input_handler(GIOChannel *channel, GIOCondition condition,
293+
gpointer data) {
294+
char buf[MAX_STRING_LEN];
295+
ssize_t cnt = read(STDIN_FILENO, buf, sizeof(buf) - 1);
296+
297+
if (cnt <= 0)
298+
return TRUE; /* Keep watching */
299+
300+
buf[cnt] = '\0';
301+
302+
if (g_menu_state == CURRENT_STATUS_MAINMENU && cnt == 2) {
303+
interpret_main_menu(buf[0]);
304+
} else if (g_menu_state != CURRENT_STATUS_MAINMENU) {
305+
/* Remove newline if present */
306+
if (buf[cnt - 1] == '\n')
307+
buf[cnt - 1] = '\0';
308+
handle_special_input(buf);
309+
}
310+
311+
display_menu();
312+
return TRUE;
313+
}
314+
315+
/**
316+
* @brief Main function
317+
*/
318+
int main(int argc, char **argv) {
319+
loop = g_main_loop_new(NULL, FALSE);
320+
GIOChannel *stdin_channel = g_io_channel_unix_new(STDIN_FILENO);
321+
guint watch_id = g_io_add_watch(stdin_channel, static_cast<GIOCondition>(G_IO_IN | G_IO_HUP), input_handler, NULL);
322+
323+
display_menu();
324+
g_main_loop_run(loop);
325+
326+
if (g_prompt)
327+
ml_lxm_prompt_destroy(g_prompt);
328+
if (g_session)
329+
ml_lxm_session_destroy(g_session);
330+
if (watch_id > 0)
331+
g_source_remove(watch_id);
332+
if (stdin_channel)
333+
g_io_channel_unref(stdin_channel);
334+
335+
return 0;
336+
}

‎Tizen.platform/lxm_service/gen-ml-lxm-service-rpm.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/usr/bin/env bash
2+
3+
git init
4+
git add *
5+
git commit -m 'Initial commit'
6+
#gbs build -A armv7l --include-all --clean
7+
gbs build --include-all -A armv7l -P public_9.0_arm
8+
rm -rf .git

0 commit comments

Comments
 (0)