[Android] support flexible format

java/android/nnstreamer/src/androidTest/assets/README.txt

@@ -7,7 +7,10 @@ $ tree .
     │   ├── config_pipeline_imgclf.conf
     │   ├── config_pipeline_imgclf_key.conf
     │   ├── config_single_imgclf.conf
-    │   └── config_single_imgclf_key.conf
+    │   ├── config_single_imgclf_key.conf
+    │   └── config_single_llamacpp_async.conf
+    ├── llamacpp_data
+    │   └── tinyllama-1.1b-chat-v1.0.Q2_K.gguf
     ├── pytorch_data
     │   ├── mobilenetv2-quant_core-nnapi.pt
     │   └── orange_float.raw
@@ -28,3 +31,23 @@ $ tree .
         ├── orange.png
         ├── orange.raw
         └── test_video.mp4
+
+
+Configuration example:
+The configuration file is a json formatted string for ML-service feature, which describes the configuration to run an inference application with model-agnostic method.
+If you implement new Android application with ML-service and need to get a model from application's internal storage, you can set the predefined entity string '@APP_DATA_PATH@'.
+Then it will be replaced with the application's data path.
+
+Below is an example of asynchronous inference using LLaMA C++.
+
+config_single_llamacpp_async.conf
+{
+    "single" :
+    {
+        "framework" : "llamacpp",
+        "model" : ["@APP_DATA_PATH@/nnstreamer/llamacpp_data/tinyllama-1.1b-chat-v1.0.Q2_K.gguf"],
+        "custom" : "num_predict:32",
+        "invoke_dynamic" : "true",
+        "invoke_async" : "true"
+    }
+}

java/android/nnstreamer/src/androidTest/java/org/nnsuite/nnstreamer/APITestMLService.java

@@ -1025,4 +1025,55 @@ public void testRunSingleShotRegistered() {
             MLService.Model.delete(name, 0);
         }
     }
+
+    @Test
+    public void testRunAsyncInvoke() {
+        if (!NNStreamer.isAvailable(NNStreamer.NNFWType.LLAMACPP)) {
+            /* cannot run the test */
+            return;
+        }
+
+        String config = APITestCommon.getConfigPath() + "/config_single_llamacpp_async.conf";
+
+        try {
+            MLService.EventListener asyncEventListener = new MLService.EventListener() {
+                @Override
+                public void onNewDataReceived(String name, TensorsData data) {
+                    if (data == null || data.getTensorsCount() != 1) {
+                        mInvalidState = true;
+                        return;
+                    }
+
+                    mReceived++;
+                }
+            };
+
+            MLService service = new MLService(config, asyncEventListener);
+
+            service.start();
+
+            /* push input buffer */
+            String inputText = "Hello my name is";
+            ByteBuffer buffer = TensorsData.allocateByteBuffer(inputText);
+
+            TensorsInfo info = service.getInputInformation(null);
+            assertEquals(NNStreamer.TensorFormat.FLEXIBLE, info.getFormat());
+
+            TensorsData input = TensorsData.allocate(info);
+            input.setTensorData(0, buffer);
+
+            service.inputData(null, input);
+
+            /* sleep 3 seconds to invoke */
+            Thread.sleep(3000);
+
+            /* check received data from output node */
+            assertFalse(mInvalidState);
+            assertTrue(mReceived > 1);
+
+            service.close();
+        } catch (Exception e) {
+            fail();
+        }
+    }
 }

java/android/nnstreamer/src/androidTest/java/org/nnsuite/nnstreamer/APITestTensorsInfo.java

@@ -30,6 +30,9 @@ public void tearDown() {
     @Test
     public void testAddInfo() {
         try {
+            /* Default format is static. */
+            assertEquals(NNStreamer.TensorFormat.STATIC, mInfo.getFormat());
+
             mInfo.addTensorInfo("name1", NNStreamer.TensorType.INT8, new int[]{1});
             assertEquals(1, mInfo.getTensorsCount());
 
@@ -251,4 +254,14 @@ public void testAddMaxInfo_n() {
             /* expected */
         }
     }
+
+    @Test
+    public void testInvalidFormat_n() {
+        try {
+            new TensorsInfo(NNStreamer.TensorFormat.UNKNOWN);
+            fail();
+        } catch (Exception e) {
+            /* expected */
+        }
+    }
 }

java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/NNStreamer.java

@@ -118,6 +118,15 @@ public enum TensorType {
         /** Unknown data type (usually error) */ UNKNOWN
     }
 
+    /**
+     * The enumeration for possible data format of tensor in NNStreamer.
+     */
+    public enum TensorFormat {
+        /** Static tensor stream */ STATIC,
+        /** Flexible tensor stream */ FLEXIBLE,
+        /** Unknown data format (usually error) */ UNKNOWN
+    }
+
     private static native boolean nativeInitialize(Object app);
     private static native boolean nativeCheckNNFWAvailability(int fw);
     private static native String nativeGetVersion();

java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/TensorsData.java

@@ -90,6 +90,7 @@ public static TensorsData allocate(TensorsInfo info) {
         int count = info.getTensorsCount();
 
         for (int i = 0; i < count; i++) {
+            /* If tensor format is flexible, data size would be 0. */
             data.addTensorData(allocateBuffer(info.getTensorSize(i)));
         }
 
@@ -181,6 +182,15 @@ private Object[] getDataArray() {
         return mDataList.toArray();
     }
 
+    /**
+     * Internal method called from native to reallocate buffer.
+     */
+    private void updateData(int index, int size) {
+        checkIndexBounds(index);
+
+        mDataList.set(index, allocateByteBuffer(size));
+    }
+
     /**
      * Internal method to check the index.
      *
@@ -224,10 +234,15 @@ private void checkByteBuffer(int index, ByteBuffer data) {
                 throw new IndexOutOfBoundsException("Current information has " + count + " tensors");
             }
 
-            int size = mInfo.getTensorSize(index);
+            NNStreamer.TensorFormat format = mInfo.getFormat();
+
+            /* The size of input buffer should be matched if data format is static. */
+            if (format == NNStreamer.TensorFormat.STATIC) {
+                int size = mInfo.getTensorSize(index);
 
-            if (data.capacity() != size) {
-                throw new IllegalArgumentException("Invalid buffer size, required size is " + size);
+                if (data.capacity() != size) {
+                    throw new IllegalArgumentException("Invalid buffer size, required size is " + size);
+                }
             }
         }
     }

java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/TensorsInfo.java

@@ -14,10 +14,42 @@
  *
  * @see NNStreamer#TENSOR_RANK_LIMIT
  * @see NNStreamer#TENSOR_SIZE_LIMIT
+ * @see NNStreamer.TensorFormat
  * @see NNStreamer.TensorType
  */
 public final class TensorsInfo implements AutoCloseable, Cloneable {
     private ArrayList<TensorInfo> mInfoList = new ArrayList<>();
+    private NNStreamer.TensorFormat mFormat;
+
+    /**
+     * Creates a new {@link TensorsInfo} instance.
+     * Default tensor format is static.
+     */
+    public TensorsInfo() {
+        this(NNStreamer.TensorFormat.STATIC);
+    }
+
+    /**
+     * Creates a new {@link TensorsInfo} instance with given tensor format.
+     *
+     * @param format The format of tensors information
+     *
+     * @throws IllegalArgumentException if tensor format is invalid
+     */
+    public TensorsInfo(NNStreamer.TensorFormat format) {
+        if (format == NNStreamer.TensorFormat.UNKNOWN) {
+            throw new IllegalArgumentException("Given tensor format is invalid");
+        }
+
+        mFormat = format;
+    }
+
+    /**
+     * Private constructor to convert tensor format value and create a new instance.
+     */
+    private TensorsInfo(int value) {
+        this(convertFormat(value));
+    }
 
     /**
      * Allocates a new {@link TensorsData} instance with the tensors information.
@@ -41,7 +73,7 @@ public TensorsData allocate() {
      */
     @Override
     public TensorsInfo clone() {
-        TensorsInfo cloned = new TensorsInfo();
+        TensorsInfo cloned = new TensorsInfo(mFormat);
 
         for (TensorInfo info : mInfoList) {
             cloned.addTensorInfo(info.getName(), info.getType(), info.getDimension());
@@ -50,6 +82,15 @@ public TensorsInfo clone() {
         return cloned;
     }
 
+    /**
+     * Gets the format of tensors information.
+     *
+     * @return The format of tensors information
+     */
+    public NNStreamer.TensorFormat getFormat() {
+        return mFormat;
+    }
+
     /**
      * Gets the number of tensors.
      * The maximum number of tensors is {@link NNStreamer#TENSOR_SIZE_LIMIT}.
@@ -90,7 +131,7 @@ public void addTensorInfo(String name, NNStreamer.TensorType type, int[] dimensi
             throw new IndexOutOfBoundsException("Max number of the tensors is " + NNStreamer.TENSOR_SIZE_LIMIT);
         }
 
-        mInfoList.add(new TensorInfo(name, type, dimension));
+        mInfoList.add(new TensorInfo(mFormat, name, type, dimension));
     }
 
     /**
@@ -190,8 +231,12 @@ public int getTensorSize(int index) {
         checkIndexBounds(index);
 
         int size = mInfoList.get(index).getSize();
-        if (size <= 0) {
-            throw new IllegalStateException("Unknown data type or invalid dimension");
+
+        /* The tensor size should be a positive value if data format is static. */
+        if (mFormat == NNStreamer.TensorFormat.STATIC) {
+            if (size <= 0) {
+                throw new IllegalStateException("Unknown data type or invalid dimension");
+            }
         }
 
         return size;
@@ -211,6 +256,34 @@ private Object[] getInfoArray() {
         return mInfoList.toArray();
     }
 
+    /**
+     * Internal method called from native to get the data format as integer value.
+     */
+    private int getFormatValue() {
+        return mFormat.ordinal();
+    }
+
+    /**
+     * Internal method to get the tensor format from int value.
+     */
+    private static NNStreamer.TensorFormat convertFormat(int value) {
+        NNStreamer.TensorFormat format = NNStreamer.TensorFormat.UNKNOWN;
+
+        switch (value) {
+            case 0:
+                format = NNStreamer.TensorFormat.STATIC;
+                break;
+            case 1:
+                format = NNStreamer.TensorFormat.FLEXIBLE;
+                break;
+            default:
+                /* unknown tensor format */
+                break;
+        }
+
+        return format;
+    }
+
     /**
      * Internal method to check the index.
      *
@@ -231,11 +304,14 @@ public void close() {
      * Internal class for tensor information.
      */
     private static class TensorInfo {
+        private NNStreamer.TensorFormat format;
         private String name = null;
         private int type = NNStreamer.TensorType.UNKNOWN.ordinal();
         private int[] dimension = new int[NNStreamer.TENSOR_RANK_LIMIT];
 
-        public TensorInfo(String name, NNStreamer.TensorType type, int[] dimension) {
+        public TensorInfo(NNStreamer.TensorFormat format, String name, NNStreamer.TensorType type, int[] dimension) {
+            this.format = format;
+
             setName(name);
             setType(type);
             setDimension(dimension);
@@ -251,7 +327,10 @@ public String getName() {
 
         public void setType(NNStreamer.TensorType type) {
             if (type == NNStreamer.TensorType.UNKNOWN) {
-                throw new IllegalArgumentException("Given tensor type is unknown or unsupported type");
+                /* The tensor type should be valid if data format is static. */
+                if (format == NNStreamer.TensorFormat.STATIC) {
+                    throw new IllegalArgumentException("Given tensor type is unknown or unsupported type");
+                }
             }
 
             this.type = type.ordinal();
@@ -281,7 +360,10 @@ public void setDimension(int[] dimension) {
             if (rank > 0) {
                 System.arraycopy(dimension, 0, this.dimension, 0, rank);
             } else {
-                throw new IllegalArgumentException("The rank of given dimension is 0");
+                /* The tensor dimension should be valid if data format is static. */
+                if (format == NNStreamer.TensorFormat.STATIC) {
+                    throw new IllegalArgumentException("The rank of given dimension is 0");
+                }
             }
 
             /* fill default value */