diff --git a/android/src/main/java/com/azihsoyn/flutter/mlkit/MlkitPlugin.java b/android/src/main/java/com/azihsoyn/flutter/mlkit/MlkitPlugin.java index 44b60ab..af647f9 100644 --- a/android/src/main/java/com/azihsoyn/flutter/mlkit/MlkitPlugin.java +++ b/android/src/main/java/com/azihsoyn/flutter/mlkit/MlkitPlugin.java @@ -111,6 +111,42 @@ public void onFailure(@NonNull Exception e) { Log.e("error", e.getMessage()); return; } + } else if (call.method.equals("FirebaseVisionTextDetector#detectFromBytes")) { + byte[] bytes = call.argument("bytes"); + int width = call.argument("width"); + int height = call.argument("height"); + try { + Bitmap bmp = BitmapFactory.decodeByteArray(bytes, 0, bytes.length); + FirebaseVisionImageMetadata meta = new FirebaseVisionImageMetadata.Builder() + .setWidth(width) + .setHeight(height) + .setFormat(FirebaseVisionImageMetadata.IMAGE_FORMAT_YV12) + //.setRotation(rotation) + .build(); + FirebaseVisionImage image = FirebaseVisionImage.fromByteArray(bytes, meta); + //FirebaseVisionImage image = FirebaseVisionImage.fromBitmap(bmp); + FirebaseVisionTextDetector detector = FirebaseVision.getInstance() + .getVisionTextDetector(); + detector.detectInImage(image) + .addOnSuccessListener( + new OnSuccessListener() { + @Override + public void onSuccess(FirebaseVisionText texts) { + result.success(processTextRecognitionResult(texts)); + } + }) + .addOnFailureListener( + new OnFailureListener() { + @Override + public void onFailure(@NonNull Exception e) { + // Task failed with an exception + e.printStackTrace(); + } + }); + } catch (Exception e) { + Log.e("error", e.getMessage()); + return; + } } else { result.notImplemented(); } diff --git a/example/android/app/build.gradle b/example/android/app/build.gradle index 3545a5e..8fea1a7 100644 --- a/example/android/app/build.gradle +++ b/example/android/app/build.gradle @@ -24,7 +24,7 @@ android { defaultConfig { // TODO: Specify your own unique Application ID (https://developer.android.com/studio/build/application-id.html). applicationId "com.azihsoyn.flutter.mlkitexample" - minSdkVersion 16 + minSdkVersion 21 targetSdkVersion 27 versionCode 1 versionName "1.0" diff --git a/example/lib/main.dart b/example/lib/main.dart index abeea70..6ccea6a 100644 --- a/example/lib/main.dart +++ b/example/lib/main.dart @@ -1,17 +1,43 @@ import 'dart:async'; import 'dart:io'; +import 'dart:typed_data'; +import 'dart:ui' as ui; +import 'package:camera/camera.dart'; import 'package:flutter/material.dart'; -import 'package:image_picker/image_picker.dart'; +import 'package:flutter/rendering.dart'; import 'package:mlkit/mlkit.dart'; +import 'package:path_provider/path_provider.dart'; +import 'package:video_player/video_player.dart'; -void main() => runApp(new MyApp()); +Future main() async { + // Fetch the available cameras before initializing the app. + try { + cameras = await availableCameras(); + } on CameraException catch (e) { + print("${e.code}, ${e.description}"); + } + runApp(MyApp()); +} class MyApp extends StatefulWidget { @override _MyAppState createState() => new _MyAppState(); } +/// Returns a suitable camera icon for [direction]. +IconData getCameraLensIcon(CameraLensDirection direction) { + switch (direction) { + case CameraLensDirection.back: + return Icons.camera_rear; + case CameraLensDirection.front: + return Icons.camera_front; + case CameraLensDirection.external: + return Icons.camera; + } + throw new ArgumentError('Unknown lens direction'); +} + class _MyAppState extends State { File _file; List _currentLabels = []; @@ -23,6 +49,30 @@ class _MyAppState extends State { super.initState(); } + GlobalKey globalKey = new GlobalKey(); + Future _capturePng() async { + print("piyo1"); + try { + RenderRepaintBoundary boundary = + globalKey.currentContext.findRenderObject(); + print("piyo2"); + ui.Image image = await boundary.toImage(); + print("piyo3"); + ByteData byteData = + await image.toByteData(format: ui.ImageByteFormat.rawRgba); + print("piyo4"); + + Uint8List pngBytes = byteData.buffer.asUint8List(); + print("piyo5"); + print(pngBytes); + print("piyo6"); + await detector.detectFromBuffer(pngBytes, image.height, image.width); + print("piyo7"); + } catch (e) { + print("piyo error ${e.toString()}"); + } + } + @override Widget build(BuildContext context) { return new MaterialApp( @@ -30,25 +80,22 @@ class _MyAppState extends State { appBar: new AppBar( title: new Text('Plugin example app'), ), - body: _buildBody(), + body: RepaintBoundary(key: globalKey, child: CameraExampleHome()), floatingActionButton: new FloatingActionButton( onPressed: () async { try { + print("fuga1"); //var file = await ImagePicker.pickImage(source: ImageSource.camera); - var file = - await ImagePicker.pickImage(source: ImageSource.gallery); - setState(() { - _file = file; - }); try { - var currentLabels = await detector.detectFromPath(_file?.path); - setState(() { - _currentLabels = currentLabels; - }); + print("fuga2"); + _capturePng(); + print("fuga3"); } catch (e) { + print("fuga4"); print(e.toString()); } } catch (e) { + print("fuga5"); print(e.toString()); } }, @@ -126,6 +173,366 @@ class _MyAppState extends State { } } +class CameraExampleHome extends StatefulWidget { + @override + _CameraExampleHomeState createState() { + return new _CameraExampleHomeState(); + } +} + +class _CameraExampleHomeState extends State { + CameraController controller; + String imagePath; + String videoPath; + VideoPlayerController videoController; + VoidCallback videoPlayerListener; + + final GlobalKey _scaffoldKey = new GlobalKey(); + + @override + Widget build(BuildContext context) { + return new Scaffold( + key: _scaffoldKey, + appBar: new AppBar( + title: const Text('Camera example'), + ), + body: new Column( + children: [ + new Expanded( + child: new Container( + child: new Padding( + padding: const EdgeInsets.all(1.0), + child: new Center( + child: _cameraPreviewWidget(), + ), + ), + decoration: new BoxDecoration( + color: Colors.black, + border: new Border.all( + color: controller != null && controller.value.isRecordingVideo + ? Colors.redAccent + : Colors.grey, + width: 3.0, + ), + ), + ), + ), + _captureControlRowWidget(), + new Padding( + padding: const EdgeInsets.all(5.0), + child: new Row( + mainAxisAlignment: MainAxisAlignment.start, + children: [ + _cameraTogglesRowWidget(), + _thumbnailWidget(), + ], + ), + ), + ], + ), + ); + } + + /// Display the preview from the camera (or a message if the preview is not available). + Widget _cameraPreviewWidget() { + if (controller == null || !controller.value.isInitialized) { + return const Text( + 'Tap a camera', + style: const TextStyle( + color: Colors.white, + fontSize: 24.0, + fontWeight: FontWeight.w900, + ), + ); + } else { + return new AspectRatio( + aspectRatio: controller.value.aspectRatio, + child: new CameraPreview(controller), + ); + } + } + + /// Display the thumbnail of the captured image or video. + Widget _thumbnailWidget() { + return new Expanded( + child: new Align( + alignment: Alignment.centerRight, + child: videoController == null && imagePath == null + ? null + : new SizedBox( + child: (videoController == null) + ? new Image.file(new File(imagePath)) + : new Container( + child: new Center( + child: new AspectRatio( + aspectRatio: videoController.value.size != null + ? videoController.value.aspectRatio + : 1.0, + child: new VideoPlayer(videoController)), + ), + decoration: new BoxDecoration( + border: new Border.all(color: Colors.pink)), + ), + width: 64.0, + height: 64.0, + ), + ), + ); + } + + /// Display the control bar with buttons to take pictures and record videos. + Widget _captureControlRowWidget() { + return new Row( + mainAxisAlignment: MainAxisAlignment.spaceEvenly, + mainAxisSize: MainAxisSize.max, + children: [ + new IconButton( + icon: const Icon(Icons.camera_alt), + color: Colors.blue, + onPressed: controller != null && + controller.value.isInitialized && + !controller.value.isRecordingVideo + ? onTakePictureButtonPressed + : null, + ), + new IconButton( + icon: const Icon(Icons.videocam), + color: Colors.blue, + onPressed: controller != null && + controller.value.isInitialized && + !controller.value.isRecordingVideo + ? onVideoRecordButtonPressed + : null, + ), + new IconButton( + icon: const Icon(Icons.stop), + color: Colors.red, + onPressed: controller != null && + controller.value.isInitialized && + controller.value.isRecordingVideo + ? onStopButtonPressed + : null, + ) + ], + ); + } + + /// Display a row of toggle to select the camera (or a message if no camera is available). + Widget _cameraTogglesRowWidget() { + final List toggles = []; + + if (cameras.isEmpty) { + return const Text('No camera found'); + } else { + for (CameraDescription cameraDescription in cameras) { + toggles.add( + new SizedBox( + width: 90.0, + child: new RadioListTile( + title: + new Icon(getCameraLensIcon(cameraDescription.lensDirection)), + groupValue: controller?.description, + value: cameraDescription, + onChanged: controller != null && controller.value.isRecordingVideo + ? null + : onNewCameraSelected, + ), + ), + ); + } + } + + return new Row(children: toggles); + } + + String timestamp() => new DateTime.now().millisecondsSinceEpoch.toString(); + + void showInSnackBar(String message) { + _scaffoldKey.currentState + .showSnackBar(new SnackBar(content: new Text(message))); + } + + void onNewCameraSelected(CameraDescription cameraDescription) async { + if (controller != null) { + await controller.dispose(); + } + controller = new CameraController(cameraDescription, ResolutionPreset.high); + + // If the controller is updated then update the UI. + controller.addListener(() { + if (mounted) setState(() {}); + if (controller.value.hasError) { + showInSnackBar('Camera error ${controller.value.errorDescription}'); + } + }); + + try { + await controller.initialize(); + } on CameraException catch (e) { + _showCameraException(e); + } + + if (mounted) { + setState(() {}); + } + } + + void onTakePictureButtonPressed() { + takePicture().then((String filePath) { + if (mounted) { + setState(() { + imagePath = filePath; + videoController?.dispose(); + videoController = null; + }); + if (filePath != null) showInSnackBar('Picture saved to $filePath'); + } + }); + } + + void onVideoRecordButtonPressed() { + startVideoRecording().then((String filePath) { + if (mounted) setState(() {}); + if (filePath != null) showInSnackBar('Saving video to $filePath'); + }); + } + + void onStopButtonPressed() { + stopVideoRecording().then((_) { + if (mounted) setState(() {}); + showInSnackBar('Video recorded to: $videoPath'); + }); + } + + Future startVideoRecording() async { + if (!controller.value.isInitialized) { + showInSnackBar('Error: select a camera first.'); + return null; + } + + final Directory extDir = await getApplicationDocumentsDirectory(); + final String dirPath = '${extDir.path}/Movies/flutter_test'; + await new Directory(dirPath).create(recursive: true); + final String filePath = '$dirPath/${timestamp()}.mp4'; + + if (controller.value.isRecordingVideo) { + // A recording is already started, do nothing. + return null; + } + + try { + videoPath = filePath; + await controller.startVideoRecording(filePath); + } on CameraException catch (e) { + _showCameraException(e); + return null; + } + return filePath; + } + + Future stopVideoRecording() async { + if (!controller.value.isRecordingVideo) { + return null; + } + + try { + await controller.stopVideoRecording(); + } on CameraException catch (e) { + _showCameraException(e); + return null; + } + + await _startVideoPlayer(); + } + + Future _startVideoPlayer() async { + final VideoPlayerController vcontroller = + new VideoPlayerController.file(new File(videoPath)); + videoPlayerListener = () { + if (videoController != null && videoController.value.size != null) { + // Refreshing the state to update video player with the correct ratio. + if (mounted) setState(() {}); + videoController.removeListener(videoPlayerListener); + } + }; + vcontroller.addListener(videoPlayerListener); + await vcontroller.setLooping(true); + await vcontroller.initialize(); + await videoController?.dispose(); + if (mounted) { + setState(() { + imagePath = null; + videoController = vcontroller; + }); + } + await vcontroller.play(); + } + + Future takePicture() async { + if (!controller.value.isInitialized) { + showInSnackBar('Error: select a camera first.'); + return null; + } + final Directory extDir = await getApplicationDocumentsDirectory(); + final String dirPath = '${extDir.path}/Pictures/flutter_test'; + await new Directory(dirPath).create(recursive: true); + final String filePath = '$dirPath/${timestamp()}.jpg'; + + if (controller.value.isTakingPicture) { + // A capture is already pending, do nothing. + return null; + } + + try { + await controller.takePicture(filePath); + } on CameraException catch (e) { + _showCameraException(e); + return null; + } + return filePath; + } + + void _showCameraException(CameraException e) { + print("${e.code}, ${e.description}"); + showInSnackBar('Error: ${e.code}\n${e.description}'); + } +} + +class PngHome extends StatefulWidget { + PngHome({Key key}) : super(key: key); + + @override + _PngHomeState createState() => new _PngHomeState(); +} + +class _PngHomeState extends State { + GlobalKey globalKey = new GlobalKey(); + + Future _capturePng() async { + RenderRepaintBoundary boundary = + globalKey.currentContext.findRenderObject(); + ui.Image image = await boundary.toImage(); + ByteData byteData = await image.toByteData(format: ui.ImageByteFormat.png); + Uint8List pngBytes = byteData.buffer.asUint8List(); + print(pngBytes); + } + + @override + Widget build(BuildContext context) { + return RepaintBoundary( + key: globalKey, + child: Center( + child: FlatButton( + child: Text('Hello World', textDirection: TextDirection.ltr), + onPressed: _capturePng, + ), + ), + ); + } +} + +List cameras; + class TextDetectDecoration extends Decoration { final Size _originalImageSize; final List _texts; diff --git a/example/pubspec.lock b/example/pubspec.lock index bd8749f..b3e51e8 100644 --- a/example/pubspec.lock +++ b/example/pubspec.lock @@ -7,28 +7,21 @@ packages: name: analyzer url: "https://pub.dartlang.org" source: hosted - version: "0.31.1" + version: "0.31.2-alpha.2" args: dependency: transitive description: name: args url: "https://pub.dartlang.org" source: hosted - version: "1.4.2" + version: "1.4.3" async: dependency: transitive description: name: async url: "https://pub.dartlang.org" source: hosted - version: "2.0.6" - barback: - dependency: transitive - description: - name: barback - url: "https://pub.dartlang.org" - source: hosted - version: "0.15.2+15" + version: "2.0.7" boolean_selector: dependency: transitive description: @@ -36,20 +29,20 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "1.0.3" - charcode: - dependency: transitive + camera: + dependency: "direct main" description: - name: charcode + name: camera url: "https://pub.dartlang.org" source: hosted - version: "1.1.1" - cli_util: + version: "0.2.1" + charcode: dependency: transitive description: - name: cli_util + name: charcode url: "https://pub.dartlang.org" source: hosted - version: "0.1.2+1" + version: "1.1.1" collection: dependency: transitive description: @@ -70,14 +63,14 @@ packages: name: crypto url: "https://pub.dartlang.org" source: hosted - version: "2.0.2+1" + version: "2.0.3" csslib: dependency: transitive description: name: csslib url: "https://pub.dartlang.org" source: hosted - version: "0.14.1" + version: "0.14.4" cupertino_icons: dependency: "direct main" description: @@ -101,7 +94,7 @@ packages: name: front_end url: "https://pub.dartlang.org" source: hosted - version: "0.1.0-alpha.9" + version: "0.1.0-alpha.12" glob: dependency: transitive description: @@ -136,7 +129,7 @@ packages: name: http_parser url: "https://pub.dartlang.org" source: hosted - version: "3.1.1" + version: "3.1.2" image_picker: dependency: "direct main" description: @@ -151,13 +144,6 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "0.3.2+1" - isolate: - dependency: transitive - description: - name: isolate - url: "https://pub.dartlang.org" - source: hosted - version: "1.1.0" js: dependency: transitive description: @@ -171,7 +157,7 @@ packages: name: kernel url: "https://pub.dartlang.org" source: hosted - version: "0.3.0-alpha.9" + version: "0.3.0-alpha.12" logging: dependency: transitive description: @@ -185,14 +171,14 @@ packages: name: matcher url: "https://pub.dartlang.org" source: hosted - version: "0.12.1+4" + version: "0.12.2+1" meta: dependency: transitive description: name: meta url: "https://pub.dartlang.org" source: hosted - version: "1.1.2" + version: "1.1.5" mime: dependency: transitive description: @@ -206,7 +192,7 @@ packages: path: ".." relative: true source: path - version: "0.0.1" + version: "0.2.2" multi_server_socket: dependency: transitive description: @@ -220,7 +206,7 @@ packages: name: node_preamble url: "https://pub.dartlang.org" source: hosted - version: "1.4.0" + version: "1.4.1" package_config: dependency: transitive description: @@ -242,6 +228,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "1.5.1" + path_provider: + dependency: "direct main" + description: + name: path_provider + url: "https://pub.dartlang.org" + source: hosted + version: "0.3.0" plugin: dependency: transitive description: @@ -262,7 +255,7 @@ packages: name: pub_semver url: "https://pub.dartlang.org" source: hosted - version: "1.3.6" + version: "1.4.1" quiver: dependency: transitive description: @@ -276,7 +269,7 @@ packages: name: shelf url: "https://pub.dartlang.org" source: hosted - version: "0.7.2" + version: "0.7.3" shelf_packages_handler: dependency: transitive description: @@ -316,7 +309,7 @@ packages: name: source_maps url: "https://pub.dartlang.org" source: hosted - version: "0.10.4" + version: "0.10.5" source_span: dependency: transitive description: @@ -337,7 +330,7 @@ packages: name: stream_channel url: "https://pub.dartlang.org" source: hosted - version: "1.6.4" + version: "1.6.6" string_scanner: dependency: transitive description: @@ -358,7 +351,7 @@ packages: name: test url: "https://pub.dartlang.org" source: hosted - version: "0.12.34" + version: "0.12.37" typed_data: dependency: transitive description: @@ -380,6 +373,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "2.0.6" + video_player: + dependency: "direct main" + description: + name: video_player + url: "https://pub.dartlang.org" + source: hosted + version: "0.5.2" watcher: dependency: transitive description: @@ -402,5 +402,5 @@ packages: source: hosted version: "2.1.13" sdks: - dart: ">=2.0.0-dev.28.0 <=2.0.0-edge.c080951d45e79cd25df98036c4be835b284a269c" - flutter: ">=0.1.4 <2.0.0" + dart: ">=2.0.0-dev.52.0 <=2.0.0-dev.54.0.flutter-46ab040e58" + flutter: ">=0.2.5 <2.0.0" diff --git a/example/pubspec.yaml b/example/pubspec.yaml index 859d24c..5f7dde8 100644 --- a/example/pubspec.yaml +++ b/example/pubspec.yaml @@ -3,6 +3,9 @@ description: Demonstrates how to use the mlkit plugin. dependencies: image_picker: ^0.4.1 + camera: ^0.2.1 + path_provider: 0.3.0 + video_player: "0.5.2" flutter: sdk: flutter diff --git a/ios/Classes/MlkitPlugin.m b/ios/Classes/MlkitPlugin.m index f4f9550..615a98e 100644 --- a/ios/Classes/MlkitPlugin.m +++ b/ios/Classes/MlkitPlugin.m @@ -26,11 +26,80 @@ - (instancetype)init { - (void)handleMethodCall:(FlutterMethodCall*)call result:(FlutterResult)result { FIRVision *vision = [FIRVision vision]; NSMutableArray *ret = [NSMutableArray array]; - NSString *path = call.arguments[@"filepath"]; - UIImage* uiImage = [UIImage imageWithContentsOfFile:path]; - FIRVisionImage *image = [[FIRVisionImage alloc] initWithImage:uiImage]; + if ([@"FirebaseVisionTextDetector#detectFromPath" isEqualToString:call.method]) { + NSString *path = call.arguments[@"filepath"]; + UIImage* uiImage = [UIImage imageWithContentsOfFile:path]; + FIRVisionImage *image = [[FIRVisionImage alloc] initWithImage:uiImage]; + textDetector = [vision textDetector]; + [textDetector detectInImage:image + completion:^(NSArray *features, + NSError *error) { + if (error != nil) { + [ret addObject:error.localizedDescription]; + result(ret); + return; + } else if (features != nil) { + // Recognized text + for (id feature in features) { + // Blocks contain lines of text + if ([feature isKindOfClass:[FIRVisionTextBlock class]]) { + FIRVisionTextBlock *block = (FIRVisionTextBlock *)feature; + [ret addObject:visionTextBlockToDictionary(block)]; + } + + // Lines contain text elements + else if ([feature isKindOfClass:[FIRVisionTextLine class]]) { + FIRVisionTextLine *line = (FIRVisionTextLine *)feature; + [ret addObject:visionTextLineToDictionary(line)]; + } + + // Text elements are typically words + else if ([feature isKindOfClass:[FIRVisionTextElement class]]) { + FIRVisionTextElement *element = (FIRVisionTextElement *)feature; + [ret addObject:visionTextElementToDictionary(element)]; + } + else { + [ret addObject:visionTextToDictionary(feature)]; + } + } + } + result(ret); + return; + }]; + } else if ([@"FirebaseVisionTextDetector#detectFromBytes" isEqualToString:call.method]) { + textDetector = [vision textDetector]; + + //NSMutableArray *bytes = [NSMutableArray array]; + const void * bytes = (__bridge const void *)(call.arguments[@"bytes"]); + + int size = call.arguments[@"length"]; + NSData* data = [NSData dataWithBytes:bytes length:size]; + //NSLog(data.base64Encoding); + + if (data == nil){ + [ret addObject:@"data is nil!!!!!!!"]; + } + UIImage* uiImage = [[UIImage alloc]initWithData:data]; + if (uiImage == nil){ + [ret addObject:@"uiImage is nil!!!!!!!"]; + } + + FIRVisionImage *image; + @try { + + image = [[FIRVisionImage alloc] initWithImage:uiImage]; + + [ret addObject:@"hge"]; + result(ret); + return; + } + + @catch (NSException *exception) { + NSLog(@"[ERROR]\nstr[%@]\nexception[%@]", exception); + } + textDetector = [vision textDetector]; [textDetector detectInImage:image completion:^(NSArray *features, @@ -68,6 +137,9 @@ - (void)handleMethodCall:(FlutterMethodCall*)call result:(FlutterResult)result { return; }]; } else if ([@"FirebaseVisionBarcodeDetector#detectFromPath" isEqualToString:call.method]) { + NSString *path = call.arguments[@"filepath"]; + UIImage* uiImage = [UIImage imageWithContentsOfFile:path]; + FIRVisionImage *image = [[FIRVisionImage alloc] initWithImage:uiImage]; /* FIRVisionBarcodeDetectorOptions *options = [[FIRVisionBarcodeDetectorOptions alloc] initWithFormats: FIRVisionBarcodeFormatAll]; diff --git a/lib/mlkit.dart b/lib/mlkit.dart index 20a25c3..6edfe74 100644 --- a/lib/mlkit.dart +++ b/lib/mlkit.dart @@ -1,5 +1,6 @@ import 'dart:async'; import 'dart:math'; +import 'dart:typed_data'; import 'dart:ui'; import 'package:flutter/services.dart'; @@ -83,6 +84,30 @@ class FirebaseVisionTextDetector { }); return ret; } + + Future detectFromBuffer(Uint8List bytes, int width, int height) async { + print("hoge1"); + try { + print("hoge2"); + print("size : ${bytes.length}, width : ${width}, height : ${height}"); + List ret = await _channel.invokeMethod( + "FirebaseVisionTextDetector#detectFromBytes", { + 'bytes': bytes, + 'length': bytes.length, + 'width': width, + 'height': height + }); + ret.forEach((dynamic item) { + print(item); + }); + print("ret : ${ret}"); + print("hoge3"); + return; + } catch (e) { + print("hoge5"); + print(e.toString()); + } + } } class FirebaseVisionBarcodeDetector {