diff --git a/java/android/nnstreamer/src/androidTest/assets/README.txt b/java/android/nnstreamer/src/androidTest/assets/README.txt index b9a0da46..e885b61f 100644 --- a/java/android/nnstreamer/src/androidTest/assets/README.txt +++ b/java/android/nnstreamer/src/androidTest/assets/README.txt @@ -7,7 +7,10 @@ $ tree . │   ├── config_pipeline_imgclf.conf │   ├── config_pipeline_imgclf_key.conf │   ├── config_single_imgclf.conf - │   └── config_single_imgclf_key.conf + │   ├── config_single_imgclf_key.conf + │   └── config_single_llamacpp_async.conf + ├── llamacpp_data + │   └── tinyllama-1.1b-chat-v1.0.Q2_K.gguf ├── pytorch_data │   ├── mobilenetv2-quant_core-nnapi.pt │   └── orange_float.raw @@ -28,3 +31,23 @@ $ tree . ├── orange.png ├── orange.raw └── test_video.mp4 + + +Configuration example: +The configuration file is a json formatted string for ML-service feature, which describes the configuration to run an inference application with model-agnostic method. +If you implement new Android application with ML-service and need to get a model from application's internal storage, you can set the predefined entity string '@APP_DATA_PATH@'. +Then it will be replaced with the application's data path. + +Below is an example of asynchronous inference using LLaMA C++. + +config_single_llamacpp_async.conf +{ + "single" : + { + "framework" : "llamacpp", + "model" : ["@APP_DATA_PATH@/nnstreamer/llamacpp_data/tinyllama-1.1b-chat-v1.0.Q2_K.gguf"], + "custom" : "num_predict:32", + "invoke_dynamic" : "true", + "invoke_async" : "true" + } +} diff --git a/java/android/nnstreamer/src/androidTest/java/org/nnsuite/nnstreamer/APITestMLService.java b/java/android/nnstreamer/src/androidTest/java/org/nnsuite/nnstreamer/APITestMLService.java index be79e26e..2baffeaa 100644 --- a/java/android/nnstreamer/src/androidTest/java/org/nnsuite/nnstreamer/APITestMLService.java +++ b/java/android/nnstreamer/src/androidTest/java/org/nnsuite/nnstreamer/APITestMLService.java @@ -1025,4 +1025,55 @@ public void testRunSingleShotRegistered() { MLService.Model.delete(name, 0); } } + + @Test + public void testRunAsyncInvoke() { + if (!NNStreamer.isAvailable(NNStreamer.NNFWType.LLAMACPP)) { + /* cannot run the test */ + return; + } + + String config = APITestCommon.getConfigPath() + "/config_single_llamacpp_async.conf"; + + try { + MLService.EventListener asyncEventListener = new MLService.EventListener() { + @Override + public void onNewDataReceived(String name, TensorsData data) { + if (data == null || data.getTensorsCount() != 1) { + mInvalidState = true; + return; + } + + mReceived++; + } + }; + + MLService service = new MLService(config, asyncEventListener); + + service.start(); + + /* push input buffer */ + String inputText = "Hello my name is"; + ByteBuffer buffer = TensorsData.allocateByteBuffer(inputText); + + TensorsInfo info = service.getInputInformation(null); + assertEquals(NNStreamer.TensorFormat.FLEXIBLE, info.getFormat()); + + TensorsData input = TensorsData.allocate(info); + input.setTensorData(0, buffer); + + service.inputData(null, input); + + /* sleep 3 seconds to invoke */ + Thread.sleep(3000); + + /* check received data from output node */ + assertFalse(mInvalidState); + assertTrue(mReceived > 1); + + service.close(); + } catch (Exception e) { + fail(); + } + } } diff --git a/java/android/nnstreamer/src/androidTest/java/org/nnsuite/nnstreamer/APITestTensorsInfo.java b/java/android/nnstreamer/src/androidTest/java/org/nnsuite/nnstreamer/APITestTensorsInfo.java index c79f9156..bac01bb1 100644 --- a/java/android/nnstreamer/src/androidTest/java/org/nnsuite/nnstreamer/APITestTensorsInfo.java +++ b/java/android/nnstreamer/src/androidTest/java/org/nnsuite/nnstreamer/APITestTensorsInfo.java @@ -30,6 +30,9 @@ public void tearDown() { @Test public void testAddInfo() { try { + /* Default format is static. */ + assertEquals(NNStreamer.TensorFormat.STATIC, mInfo.getFormat()); + mInfo.addTensorInfo("name1", NNStreamer.TensorType.INT8, new int[]{1}); assertEquals(1, mInfo.getTensorsCount()); @@ -251,4 +254,14 @@ public void testAddMaxInfo_n() { /* expected */ } } + + @Test + public void testInvalidFormat_n() { + try { + new TensorsInfo(NNStreamer.TensorFormat.UNKNOWN); + fail(); + } catch (Exception e) { + /* expected */ + } + } } diff --git a/java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/NNStreamer.java b/java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/NNStreamer.java index 121dfff2..4bc7230c 100644 --- a/java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/NNStreamer.java +++ b/java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/NNStreamer.java @@ -118,6 +118,15 @@ public enum TensorType { /** Unknown data type (usually error) */ UNKNOWN } + /** + * The enumeration for possible data format of tensor in NNStreamer. + */ + public enum TensorFormat { + /** Static tensor stream */ STATIC, + /** Flexible tensor stream */ FLEXIBLE, + /** Unknown data format (usually error) */ UNKNOWN + } + private static native boolean nativeInitialize(Object app); private static native boolean nativeCheckNNFWAvailability(int fw); private static native String nativeGetVersion(); diff --git a/java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/TensorsData.java b/java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/TensorsData.java index e3e081c1..1cb8b0c0 100644 --- a/java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/TensorsData.java +++ b/java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/TensorsData.java @@ -90,6 +90,7 @@ public static TensorsData allocate(TensorsInfo info) { int count = info.getTensorsCount(); for (int i = 0; i < count; i++) { + /* If tensor format is flexible, data size would be 0. */ data.addTensorData(allocateBuffer(info.getTensorSize(i))); } @@ -181,6 +182,15 @@ private Object[] getDataArray() { return mDataList.toArray(); } + /** + * Internal method called from native to reallocate buffer. + */ + private void updateData(int index, int size) { + checkIndexBounds(index); + + mDataList.set(index, allocateByteBuffer(size)); + } + /** * Internal method to check the index. * @@ -224,10 +234,15 @@ private void checkByteBuffer(int index, ByteBuffer data) { throw new IndexOutOfBoundsException("Current information has " + count + " tensors"); } - int size = mInfo.getTensorSize(index); + NNStreamer.TensorFormat format = mInfo.getFormat(); + + /* The size of input buffer should be matched if data format is static. */ + if (format == NNStreamer.TensorFormat.STATIC) { + int size = mInfo.getTensorSize(index); - if (data.capacity() != size) { - throw new IllegalArgumentException("Invalid buffer size, required size is " + size); + if (data.capacity() != size) { + throw new IllegalArgumentException("Invalid buffer size, required size is " + size); + } } } } diff --git a/java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/TensorsInfo.java b/java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/TensorsInfo.java index 38215197..b3c71232 100644 --- a/java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/TensorsInfo.java +++ b/java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/TensorsInfo.java @@ -14,10 +14,42 @@ * * @see NNStreamer#TENSOR_RANK_LIMIT * @see NNStreamer#TENSOR_SIZE_LIMIT + * @see NNStreamer.TensorFormat * @see NNStreamer.TensorType */ public final class TensorsInfo implements AutoCloseable, Cloneable { private ArrayList mInfoList = new ArrayList<>(); + private NNStreamer.TensorFormat mFormat; + + /** + * Creates a new {@link TensorsInfo} instance. + * Default tensor format is static. + */ + public TensorsInfo() { + this(NNStreamer.TensorFormat.STATIC); + } + + /** + * Creates a new {@link TensorsInfo} instance with given tensor format. + * + * @param format The format of tensors information + * + * @throws IllegalArgumentException if tensor format is invalid + */ + public TensorsInfo(NNStreamer.TensorFormat format) { + if (format == NNStreamer.TensorFormat.UNKNOWN) { + throw new IllegalArgumentException("Given tensor format is invalid"); + } + + mFormat = format; + } + + /** + * Private constructor to convert tensor format value and create a new instance. + */ + private TensorsInfo(int value) { + this(convertFormat(value)); + } /** * Allocates a new {@link TensorsData} instance with the tensors information. @@ -41,7 +73,7 @@ public TensorsData allocate() { */ @Override public TensorsInfo clone() { - TensorsInfo cloned = new TensorsInfo(); + TensorsInfo cloned = new TensorsInfo(mFormat); for (TensorInfo info : mInfoList) { cloned.addTensorInfo(info.getName(), info.getType(), info.getDimension()); @@ -50,6 +82,15 @@ public TensorsInfo clone() { return cloned; } + /** + * Gets the format of tensors information. + * + * @return The format of tensors information + */ + public NNStreamer.TensorFormat getFormat() { + return mFormat; + } + /** * Gets the number of tensors. * The maximum number of tensors is {@link NNStreamer#TENSOR_SIZE_LIMIT}. @@ -90,7 +131,7 @@ public void addTensorInfo(String name, NNStreamer.TensorType type, int[] dimensi throw new IndexOutOfBoundsException("Max number of the tensors is " + NNStreamer.TENSOR_SIZE_LIMIT); } - mInfoList.add(new TensorInfo(name, type, dimension)); + mInfoList.add(new TensorInfo(mFormat, name, type, dimension)); } /** @@ -190,8 +231,12 @@ public int getTensorSize(int index) { checkIndexBounds(index); int size = mInfoList.get(index).getSize(); - if (size <= 0) { - throw new IllegalStateException("Unknown data type or invalid dimension"); + + /* The tensor size should be a positive value if data format is static. */ + if (mFormat == NNStreamer.TensorFormat.STATIC) { + if (size <= 0) { + throw new IllegalStateException("Unknown data type or invalid dimension"); + } } return size; @@ -211,6 +256,34 @@ private Object[] getInfoArray() { return mInfoList.toArray(); } + /** + * Internal method called from native to get the data format as integer value. + */ + private int getFormatValue() { + return mFormat.ordinal(); + } + + /** + * Internal method to get the tensor format from int value. + */ + private static NNStreamer.TensorFormat convertFormat(int value) { + NNStreamer.TensorFormat format = NNStreamer.TensorFormat.UNKNOWN; + + switch (value) { + case 0: + format = NNStreamer.TensorFormat.STATIC; + break; + case 1: + format = NNStreamer.TensorFormat.FLEXIBLE; + break; + default: + /* unknown tensor format */ + break; + } + + return format; + } + /** * Internal method to check the index. * @@ -231,11 +304,14 @@ public void close() { * Internal class for tensor information. */ private static class TensorInfo { + private NNStreamer.TensorFormat format; private String name = null; private int type = NNStreamer.TensorType.UNKNOWN.ordinal(); private int[] dimension = new int[NNStreamer.TENSOR_RANK_LIMIT]; - public TensorInfo(String name, NNStreamer.TensorType type, int[] dimension) { + public TensorInfo(NNStreamer.TensorFormat format, String name, NNStreamer.TensorType type, int[] dimension) { + this.format = format; + setName(name); setType(type); setDimension(dimension); @@ -251,7 +327,10 @@ public String getName() { public void setType(NNStreamer.TensorType type) { if (type == NNStreamer.TensorType.UNKNOWN) { - throw new IllegalArgumentException("Given tensor type is unknown or unsupported type"); + /* The tensor type should be valid if data format is static. */ + if (format == NNStreamer.TensorFormat.STATIC) { + throw new IllegalArgumentException("Given tensor type is unknown or unsupported type"); + } } this.type = type.ordinal(); @@ -281,7 +360,10 @@ public void setDimension(int[] dimension) { if (rank > 0) { System.arraycopy(dimension, 0, this.dimension, 0, rank); } else { - throw new IllegalArgumentException("The rank of given dimension is 0"); + /* The tensor dimension should be valid if data format is static. */ + if (format == NNStreamer.TensorFormat.STATIC) { + throw new IllegalArgumentException("The rank of given dimension is 0"); + } } /* fill default value */ diff --git a/java/android/nnstreamer/src/main/jni/nnstreamer-native-api.c b/java/android/nnstreamer/src/main/jni/nnstreamer-native-api.c index 7847a384..f219d0d4 100644 --- a/java/android/nnstreamer/src/main/jni/nnstreamer-native-api.c +++ b/java/android/nnstreamer/src/main/jni/nnstreamer-native-api.c @@ -124,6 +124,8 @@ nns_construct_tensors_data_cls_info (JNIEnv * env, "()[Ljava/lang/Object;"); info->mid_get_info = (*env)->GetMethodID (env, info->cls, "getTensorsInfo", "()L" NNS_CLS_TINFO ";"); + info->mid_update_data = (*env)->GetMethodID (env, info->cls, "updateData", + "(II)V"); } /** @@ -154,11 +156,13 @@ nns_construct_tensors_info_cls_info (JNIEnv * env, info->cls_info = (*env)->NewGlobalRef (env, cls); (*env)->DeleteLocalRef (env, cls); - info->mid_init = (*env)->GetMethodID (env, info->cls, "", "()V"); + info->mid_init = (*env)->GetMethodID (env, info->cls, "", "(I)V"); info->mid_add_info = (*env)->GetMethodID (env, info->cls, "appendInfo", "(Ljava/lang/String;I[I)V"); info->mid_get_array = (*env)->GetMethodID (env, info->cls, "getInfoArray", "()[Ljava/lang/Object;"); + info->mid_get_format = (*env)->GetMethodID (env, info->cls, "getFormatValue", + "()I"); info->fid_info_name = (*env)->GetFieldID (env, info->cls_info, "name", "Ljava/lang/String;"); @@ -404,6 +408,7 @@ nns_convert_tensors_data (pipeline_info_s * pipe_info, JNIEnv * env, jobject obj_data = NULL; jobjectArray data_arr; ml_tensors_data_s *data; + ml_tensors_info_s *info; g_return_val_if_fail (pipe_info, FALSE); g_return_val_if_fail (env, FALSE); @@ -421,6 +426,42 @@ nns_convert_tensors_data (pipeline_info_s * pipe_info, JNIEnv * env, data_arr = (*env)->CallObjectMethod (env, obj_data, tensors_data_cls->mid_get_array); + /* If data format is flexible, we should check allocated buffer. */ + info = (ml_tensors_info_s *) data->info; + if (info->info.format == _NNS_TENSOR_FORMAT_FLEXIBLE) { + gboolean changed = FALSE; + + for (i = 0; i < data->num_tensors; i++) { + jobject tensor = (*env)->GetObjectArrayElement (env, data_arr, i); + size_t capacity = (size_t) (*env)->GetDirectBufferCapacity (env, tensor); + + (*env)->DeleteLocalRef (env, tensor); + + if (data->tensors[i].size != capacity) { + (*env)->CallVoidMethod (env, obj_data, + tensors_data_cls->mid_update_data, i, data->tensors[i].size); + + if ((*env)->ExceptionCheck (env)) { + _ml_loge ("Failed to update data[%u] with size %zd (old size %zd).", + i, data->tensors[i].size, capacity); + (*env)->ExceptionClear (env); + + (*env)->DeleteLocalRef (env, data_arr); + return FALSE; + } + + changed = TRUE; + } + } + + if (changed) { + (*env)->DeleteLocalRef (env, data_arr); + + data_arr = (*env)->CallObjectMethod (env, obj_data, + tensors_data_cls->mid_get_array); + } + } + for (i = 0; i < data->num_tensors; i++) { jobject tensor = (*env)->GetObjectArrayElement (env, data_arr, i); gpointer data_ptr = (*env)->GetDirectBufferAddress (env, tensor); @@ -501,6 +542,11 @@ nns_parse_tensors_data (pipeline_info_s * pipe_info, JNIEnv * env, gpointer data_ptr = (*env)->GetDirectBufferAddress (env, tensor); if (clone) { + if (data->tensors[i].data && data->tensors[i].size != data_size) { + _ml_logd ("The data size is not matched, reallocate new memory."); + g_clear_pointer (&data->tensors[i].data, g_free); + } + if (data->tensors[i].data == NULL) data->tensors[i].data = g_malloc (data_size); @@ -550,12 +596,18 @@ nns_convert_tensors_info (pipeline_info_s * pipe_info, JNIEnv * env, tensors_info_cls = &pipe_info->tensors_info_cls_info; info = (ml_tensors_info_s *) info_h; + *result = NULL; obj_info = (*env)->NewObject (env, tensors_info_cls->cls, - tensors_info_cls->mid_init); - if (!obj_info) { + tensors_info_cls->mid_init, (jint) info->info.format); + if ((*env)->ExceptionCheck (env) || !obj_info) { _ml_loge ("Failed to allocate object for tensors info."); - goto done; + (*env)->ExceptionClear (env); + + if (obj_info) + (*env)->DeleteLocalRef (env, obj_info); + + return FALSE; } for (i = 0; i < info->info.num_tensors; i++) { @@ -582,9 +634,8 @@ nns_convert_tensors_info (pipeline_info_s * pipe_info, JNIEnv * env, (*env)->DeleteLocalRef (env, dimension); } -done: *result = obj_info; - return (obj_info != NULL); + return TRUE; } /** @@ -616,6 +667,10 @@ nns_parse_tensors_info (pipeline_info_s * pipe_info, JNIEnv * env, info_arr = (*env)->CallObjectMethod (env, obj_info, tensors_info_cls->mid_get_array); + /* tensor format */ + info->info.format = (tensor_format) (*env)->CallIntMethod (env, obj_info, + tensors_info_cls->mid_get_format); + /* number of tensors info */ info->info.num_tensors = (unsigned int) (*env)->GetArrayLength (env, info_arr); diff --git a/java/android/nnstreamer/src/main/jni/nnstreamer-native-internal.h b/java/android/nnstreamer/src/main/jni/nnstreamer-native-internal.h index ecb28c81..8777fcb2 100644 --- a/java/android/nnstreamer/src/main/jni/nnstreamer-native-internal.h +++ b/java/android/nnstreamer/src/main/jni/nnstreamer-native-internal.h @@ -112,6 +112,7 @@ typedef struct jmethodID mid_alloc; jmethodID mid_get_array; jmethodID mid_get_info; + jmethodID mid_update_data; } tensors_data_class_info_s; /** @@ -123,6 +124,7 @@ typedef struct jmethodID mid_init; jmethodID mid_add_info; jmethodID mid_get_array; + jmethodID mid_get_format; jclass cls_info; jfieldID fid_info_name;