segmentation via yolov8 done

vladiH · vladiH · commit a19c14aa741b · 2023-09-16T14:12:04.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 1.1.3
+* Release of segmentation feature via YOLOv8.
+* Updated example code.
+* Updated README.
 ## 1.1.2
 * GPU delegation error has been fixed.
 * Coordinate representation of the box in documentation was fixed.
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # flutter_vision
 
-A Flutter plugin for managing [Yolov5, Yolov8](https://github.com/ultralytics/ultralytics) and [Tesseract v5](https://tesseract-ocr.github.io/tessdoc/) accessing with TensorFlow Lite 2.x. Support object detection and OCR on Android. iOS not updated, working in progress.
+A Flutter plugin for managing [Yolov5, Yolov8](https://github.com/ultralytics/ultralytics) and [Tesseract v5](https://tesseract-ocr.github.io/tessdoc/) accessing with TensorFlow Lite 2.x. Support object detection, segmentation and OCR on Android. iOS not updated, working in progress.
 
 # Installation
 Add flutter_vision as a dependency in your pubspec.yaml file.
@@ -42,7 +42,7 @@ import 'package:flutter_vision/flutter_vision.dart';
 ```
 
 4. Load the model and labels:
-`modelVersion`: yolov5 or yolov8
+`modelVersion`: yolov5 or yolov8 or yolov8seg
 ```dart
 await vision.loadYoloModel(
         labels: 'assets/labelss.txt',
@@ -68,7 +68,7 @@ final result = await vision.yoloOnFrame(
 ```
 
 ### For static image
-5. Make your first detection:
+5. Make your first detection or segmentation:
 
 ```dart
 final result = await vision.yoloOnImage(
@@ -143,13 +143,24 @@ await vision.loadTesseractModel(
 await vision.closeTesseractModel();
 ```
 # About results
-## For Yolo
+## For Yolo v5 or v8 in detection task
+result is a `List<Map<String,dynamic>>` where Map have the following keys:
+
+ ``` dart
+    Map<String, dynamic>:{
+     "box": [x1:left, y1:top, x2:right, y2:bottom, class_confidence]
+     "tag": String: detected class
+    }
+```
+
+## For YoloV8 in segmentation task
 result is a `List<Map<String,dynamic>>` where Map have the following keys:
 
  ``` dart
     Map<String, dynamic>:{
      "box": [x1:left, y1:top, x2:right, y2:bottom, class_confidence]
      "tag": String: detected class
+     "polygons": List<Map<String, double>>: [{x:coordx, y:coordy}]
     }
 ```
 
@@ -165,5 +176,17 @@ result is a `List<Map<String,dynamic>>` where Map have the following keys:
 
 # Example
 ![Screenshot_2022-04-08-23-59-05-652_com vladih dni_scanner_example](https://user-images.githubusercontent.com/32783435/164163922-2eb7c8a3-8415-491f-883e-12cc87512efe.jpg)
-![Screenshot_2022-04-08-23-59-42-594_com vladih dni_scanner_example](https://user-images.githubusercontent.com/32783435/164163927-b290e46b-2af8-4b2b-a6a4-88cf4075f388.jpg)
-![Screenshot_2022-04-09-00-00-53-316_com vladih dni_scanner_example](https://user-images.githubusercontent.com/32783435/164163929-4b22310a-e6f6-4453-886b-7c7b622892de.jpg)
+<img src="https://github.com/vladiH/flutter_vision/assets/32783435/8fbbb9da-062e-4089-b1f6-1d4b2795664c" alt="Home" width="250" height="555">
+<img src="https://github.com/vladiH/flutter_vision/assets/32783435/51098356-8ed7-4c72-bce0-cf64d56976cd" alt="Detection" width="250" height="555">
+<img src="https://github.com/vladiH/flutter_vision/assets/32783435/0368ae2f-89ad-4a1a-a4c3-0548f0948421" alt="Segmentation" width="250" height="555">
+
+
+# <div align="center">Contact</div>
+
+For flutter_vision bug reports and feature requests please visit [GitHub Issues](https://github.com/vladiH/flutter_vision/issues)
+
+<br>
+<div align="center">
+  <a href="https://linktr.ee/randpoint" style="text-decoration:none;">
+    <img src="https://github.com/vladiH/flutter_vision/assets/32783435/75d2c677-65b5-4b9d-b332-d9d08602f024" width="3%" alt="" /></a>
+</div>
diff --git a/android/src/main/java/com/vladih/computer_vision/flutter_vision/models/Yolo.java b/android/src/main/java/com/vladih/computer_vision/flutter_vision/models/Yolo.java
@@ -22,6 +22,7 @@
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.InputStreamReader;
+import java.lang.reflect.Array;
 import java.nio.ByteBuffer;
 import java.nio.MappedByteBuffer;
 import java.nio.channels.FileChannel;
@@ -118,8 +119,8 @@ public void initialize_model() throws Exception {
             }
             this.interpreter.allocateTensors();
             this.labels = load_labels(asset_manager, label_path);
-            int[] shape = interpreter.getOutputTensor(0).shape();
-            this.output = new float[shape[0]][shape[1]][shape[2]];
+            int[] shape = interpreter.getOutputTensor(0).shape();//3dimension
+            this.output = (float [][][]) Array.newInstance(float.class, shape);
         } catch (Exception e) {
             throw e;
         } finally {
@@ -161,10 +162,11 @@ public List<Map<String, Object>> detect_task(ByteBuffer byteBuffer,
                                                  float iou_threshold,
                                                  float conf_threshold, float class_threshold) throws Exception {
         try {
-            int[] shape = this.interpreter.getInputTensor(0).shape();
+            int[] input_shape = this.interpreter.getInputTensor(0).shape();
             this.interpreter.run(byteBuffer, this.output);
-            List<float[]> boxes = filter_box(this.output, iou_threshold, conf_threshold, class_threshold, shape[1], shape[2]);
-            boxes = restore_size(boxes, shape[1], shape[2], source_width, source_height);
+            List<float[]> boxes = filter_box(this.output, iou_threshold, conf_threshold,
+                    class_threshold, input_shape[1], input_shape[2]);
+            boxes = restore_size(boxes, input_shape[1], input_shape[2], source_width, source_height);
             return out(boxes, this.labels);
         } catch (Exception e) {
             throw e;
@@ -183,6 +185,8 @@ protected List<float[]> filter_box(float[][][] model_outputs, float iou_threshol
             int dimension = model_outputs[0][0].length;
             int rows = model_outputs[0].length;
             float x1, y1, x2, y2, conf;
+            int max_index = 0;
+            float max = 0f;
             for (int i = 0; i < rows; i++) {
                 //convert xywh to xyxy
                 x1 = (model_outputs[0][i][0] - model_outputs[0][i][2] / 2f) * input_width;
@@ -191,16 +195,18 @@ protected List<float[]> filter_box(float[][][] model_outputs, float iou_threshol
                 y2 = (model_outputs[0][i][1] + model_outputs[0][i][3] / 2f) * input_height;
                 conf = model_outputs[0][i][conf_index];
                 if (conf < conf_threshold) continue;
-                float max = 0;
-                int max_index = 0;
-                for (int j = class_index; j < dimension; j++) {
-                    if (model_outputs[0][i][j] < class_threshold) continue;
-                    if (max < model_outputs[0][i][j]) {
-                        max = model_outputs[0][i][j];
+
+                max_index = class_index;
+                max = model_outputs[0][i][max_index];
+
+                for (int j = class_index + 1; j < dimension; j++) {
+                    float current = model_outputs[0][i][j];
+                    if (current > max) {
+                        max = current;
                         max_index = j;
                     }
                 }
-                if (max > 0) {
+                if (max > class_threshold){
                     float[] tmp = new float[6];
                     tmp[0] = x1;
                     tmp[1] = y1;
@@ -213,7 +219,7 @@ protected List<float[]> filter_box(float[][][] model_outputs, float iou_threshol
             }
             if (pre_box.isEmpty()) return new ArrayList<>();
             //for reverse orden, insteand of using .reversed method
-            Comparator<float[]> compareValues = (v1, v2) -> Float.compare(v1[1], v2[1]);
+            Comparator<float[]> compareValues = (v1, v2) -> Float.compare(v2[4], v1[4]);
             //Collections.sort(pre_box,compareValues.reversed());
             Collections.sort(pre_box, compareValues);
             return nms(pre_box, iou_threshold);
diff --git a/android/src/main/java/com/vladih/computer_vision/flutter_vision/models/Yolov8.java b/android/src/main/java/com/vladih/computer_vision/flutter_vision/models/Yolov8.java
@@ -59,11 +59,12 @@ public List<Map<String, Object>> detect_task(ByteBuffer byteBuffer,
                                                  float conf_threshold,
                                                  float class_threshold) throws Exception {
         try {
-            int[] shape = this.interpreter.getInputTensor(0).shape();
+            int[] input_shape = this.interpreter.getInputTensor(0).shape();
             this.interpreter.run(byteBuffer, this.output);
             //INFO: output from detection model is not normalized
-            List<float[]> boxes = filter_box(this.output, iou_threshold, conf_threshold, class_threshold, shape[1], shape[2]);
-            boxes = restore_size(boxes, shape[1], shape[2], source_width, source_height);
+            List<float[]> boxes = filter_box(this.output, iou_threshold, conf_threshold,
+                    class_threshold, input_shape[1], input_shape[2]);
+            boxes = restore_size(boxes, input_shape[1], input_shape[2], source_width, source_height);
             return out(boxes, this.labels);
         } catch (Exception e) {
             throw e;
@@ -81,15 +82,16 @@ protected List<float[]> filter_box(float[][][] model_outputs, float iou_threshol
             int class_index = 4;
             int dimension = model_outputs[0][0].length;
             int rows = model_outputs[0].length;
-
+            int max_index = 0;
+            float max = 0f;
             for (int i = 0; i < dimension; i++) {
                 float x1 = (model_outputs[0][0][i] - model_outputs[0][2][i] / 2f);
                 float y1 = (model_outputs[0][1][i] - model_outputs[0][3][i] / 2f);
                 float x2 = (model_outputs[0][0][i] + model_outputs[0][2][i] / 2f);
                 float y2 = (model_outputs[0][1][i] + model_outputs[0][3][i] / 2f);
 
-                int max_index = class_index;
-                float max = model_outputs[0][max_index][i];
+                max_index = class_index;
+                max = model_outputs[0][max_index][i];
 
                 for (int j = class_index + 1; j < rows; j++) {
                     float current = model_outputs[0][j][i];
@@ -112,7 +114,7 @@ protected List<float[]> filter_box(float[][][] model_outputs, float iou_threshol
             }
             if (pre_box.isEmpty()) return new ArrayList<>();
             //for reverse orden, insteand of using .reversed method
-            Comparator<float[]> compareValues = (v1, v2) -> Float.compare(v1[1], v2[1]);
+            Comparator<float[]> compareValues = (v1, v2) -> Float.compare(v2[4], v1[4]);
             //Collections.sort(pre_box,compareValues.reversed());
             Collections.sort(pre_box, compareValues);
             return nms(pre_box, iou_threshold);
diff --git a/android/src/main/java/com/vladih/computer_vision/flutter_vision/models/Yolov8Seg.java b/android/src/main/java/com/vladih/computer_vision/flutter_vision/models/Yolov8Seg.java
@@ -90,7 +90,7 @@ public List<Map<String, Object>> detect_task(ByteBuffer byteBuffer,
                 List<int[]> seg_boxes_mask = new ArrayList<>();
                 for (float[] mask_weight : seg_boxes) {
                     seg_boxes_mask.add(compute_mask(mask_weight,
-                            masks[0], (float) 0.5,
+                            masks[0], (float) 0.3,
                             output1_shape[1], output1_shape[2]));
                 }
                 masks = null;
@@ -160,9 +160,9 @@ private List<List<Map<String, Double>>> restore_seg_mask_size(List<float[]> boxe
                         min(mask_height, Math.max(boxes.get(i)[3] * mask_height / source_height, 0))
                 );
 //            utils.getScreenshotBmp(crop, tag+"1");
-                List<List<Map<String, Double>>> crop_polygon = get_polygons_from_bitmap(crop, mask_height,
+                List<Map<String, Double>> crop_polygon = get_polygons_from_bitmap(crop, mask_height,
                         mask_width, source_height, source_width);
-                polygons.add(crop_polygon.get(0));
+                polygons.add(crop_polygon);
             }
             return polygons;
         } catch (Exception e) {
@@ -173,7 +173,7 @@ private List<List<Map<String, Double>>> restore_seg_mask_size(List<float[]> boxe
         }
     }
 
-    public static List<List<Map<String, Double>>> get_polygons_from_bitmap(Bitmap mask,
+    public static List<Map<String, Double>> get_polygons_from_bitmap(Bitmap mask,
                                                                            int mask_height,
                                                                            int mask_width,
                                                                            int source_height,
@@ -182,27 +182,39 @@ public static List<List<Map<String, Double>>> get_polygons_from_bitmap(Bitmap ma
         List<MatOfPoint> contours = new ArrayList<>();
         Imgproc.findContours(maskMat, contours, new Mat(), Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE);
 
-        List<List<Point>> polygons = new ArrayList<>();
+        MatOfPoint largestContour = null;
+        double largestArea = 0;
+
         for (MatOfPoint contour : contours) {
-            List<Point> polygon = new ArrayList<>();
-            for (Point point : contour.toList()) {
-                polygon.add(point);
+            double area = Imgproc.contourArea(contour);
+            if (area > largestArea) {
+                largestArea = area;
+                largestContour = contour;
             }
-            polygons.add(polygon);
         }
-        List<List<Map<String, Double>>> converted_polygons = new ArrayList<>();
+        List<Point> polygon = new ArrayList<>(largestContour.toList());
+//        List<List<Point>> polygons = new ArrayList<>();
+//        for (MatOfPoint contour : contours) {
+//            List<Point> polygon = new ArrayList<>();
+//            for (Point point : contour.toList()) {
+//                polygon.add(point);
+//            }
+//            polygons.add(polygon);
+//        }
+//        List<List<Map<String, Double>>> converted_polygons = new ArrayList<>();
 
-        for (List<Point> polygon : polygons) {
+//        for (List<Point> polygon : polygons) {
             List<Map<String, Double>> convertedPolygon = new ArrayList<>();
             for (Point point : polygon) {
                 Map<String, Double> pointMap = new HashMap<>();
                 pointMap.put("x", point.x * source_width / mask_width);
                 pointMap.put("y", point.y * source_height / mask_height);
                 convertedPolygon.add(pointMap);
             }
-            converted_polygons.add(convertedPolygon);
-        }
-        return converted_polygons;
+//            converted_polygons.add(convertedPolygon);
+//        }
+//        return converted_polygons;
+        return convertedPolygon;
     }
 
     private boolean has_multiple_output() {
@@ -221,16 +233,17 @@ protected List<float[]> filter_box(float[][][] model_outputs, float iou_threshol
             int rows = model_outputs[0].length;
             int index_mask = rows - 32;
             float[] mask_weight = new float[32];
-
+            int max_index = 0;
+            float max = 0f;
             for (int i = 0; i < dimension; i++) {
                 // Convertir xywh a xyxy y ajustar por el ancho y alto de entrada
                 float x1 = (model_outputs[0][0][i] - model_outputs[0][2][i] / 2f) * input_width;
                 float y1 = (model_outputs[0][1][i] - model_outputs[0][3][i] / 2f) * input_height;
                 float x2 = (model_outputs[0][0][i] + model_outputs[0][2][i] / 2f) * input_width;
                 float y2 = (model_outputs[0][1][i] + model_outputs[0][3][i] / 2f) * input_height;
 
-                int max_index = class_index;
-                float max = model_outputs[0][max_index][i];
+                max_index = class_index;
+                max = model_outputs[0][max_index][i];
 
                 for (int j = class_index + 1; j < index_mask; j++) {
                     float current = model_outputs[0][j][i];
@@ -256,7 +269,7 @@ protected List<float[]> filter_box(float[][][] model_outputs, float iou_threshol
             }
             if (pre_box.isEmpty()) return new ArrayList<>();
             //for reverse orden, insteand of using .reversed method
-            Comparator<float[]> compareValues = (v1, v2) -> Float.compare(v1[1], v2[1]);
+            Comparator<float[]> compareValues = (v1, v2) -> Float.compare(v2[4], v1[4]);
             //Collections.sort(pre_box,compareValues.reversed());
             Collections.sort(pre_box, compareValues);
             return nms(pre_box, iou_threshold);
diff --git a/example/lib/main.dart b/example/lib/main.dart
@@ -186,7 +186,7 @@ class _YoloVideoState extends State<YoloVideo> {
 
   init() async {
     cameras = await availableCameras();
-    controller = CameraController(cameras[0], ResolutionPreset.low);
+    controller = CameraController(cameras[0], ResolutionPreset.medium);
     controller.initialize().then((value) {
       loadYoloModel().then((value) {
         setState(() {
@@ -559,7 +559,7 @@ class _YoloImageV8State extends State<YoloImageV8> {
   Future<void> loadYoloModel() async {
     await widget.vision.loadYoloModel(
         labels: 'assets/labels.txt',
-        modelPath: 'assets/yolov8n-seg.tflite',
+        modelPath: 'assets/yolov8n.tflite',
         modelVersion: "yolov8",
         quantization: false,
         numThreads: 2,
diff --git a/pubspec.yaml b/pubspec.yaml
@@ -1,7 +1,7 @@
 name: flutter_vision
-description: A Flutter plugin for managing Yolov5, Yolov8 and Tesseract v5 accessing with TensorFlow Lite 2.x. Support object detection and OCR on Android. iOS not updated, working in progress.
+description: A Flutter plugin for managing Yolov5, Yolov8 and Tesseract v5 accessing with TensorFlow Lite 2.x. Support object detection, segmentation and OCR on Android. iOS not updated, working in progress.
   
-version: 1.1.2
+version: 1.1.3
 
 homepage: https://github.com/vladiH/flutter_vision
 
