Skip to content
This repository was archived by the owner on Apr 4, 2023. It is now read-only.

Commit 35c646d

Browse files
EddyVerbruggenEddyVerbruggen
committed
#699 Add ML Kit support (ML Kit demo++, features++, restored non-MLKit demo)
1 parent dba179a commit 35c646d

29 files changed

+518
-365
lines changed

‎demo-ng/app/App_Resources/Android/AndroidManifest.xml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,13 @@
2525
android:label="@string/app_name"
2626
android:theme="@style/AppTheme">
2727

28-
<meta-data android:name="com.facebook.sdk.ApplicationId" android:value="@string/facebook_app_id"/>
28+
<meta-data
29+
android:name="com.facebook.sdk.ApplicationId"
30+
android:value="@string/facebook_app_id"/>
31+
32+
<meta-data
33+
android:name="com.google.firebase.ml.vision.DEPENDENCIES"
34+
android:value="text,barcode,face,label" />
2935

3036
<activity
3137
android:name="com.tns.NativeScriptActivity"

‎demo-ng/app/app.css‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,3 +83,8 @@ button {
8383
height: 280;
8484
margin-top: 16;
8585
}
86+
87+
Label.mlkit-result {
88+
margin: 5;
89+
text-align: left;
90+
}

‎demo-ng/app/tabs/mlkit/barcodescanning/barcodescanning.component.html‎

Lines changed: 19 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -2,27 +2,26 @@
22
<Label class="action-bar-title" text="Barcode scanning"></Label>
33
</ActionBar>
44

5-
<ScrollView>
6-
<StackLayout>
5+
<GridLayout rows="auto, auto, *">
76

8-
<Label text="The scanner has been configured to detect QR codes, EAN 8 and EAN 13. It processes every 10th frame. These settings can be tweaked in your usage of the plugin." textWrap="true"></Label>
7+
<Label row="0" text="The scanner has been configured to detect QR codes, EAN 8 and EAN 13. It processes every 10th frame. These settings can be tweaked in your usage of the plugin." textWrap="true"></Label>
98

10-
<MLKitBarcodeScanner
11-
width="300"
12-
height="340"
13-
formats="QR_CODE, EAN_8, EAN_13"
14-
processEveryNthFrame="10"
15-
(scanResult)="onBarcodeScanResult($event)">
16-
</MLKitBarcodeScanner>
9+
<MLKitBarcodeScanner
10+
row="1"
11+
width="300"
12+
height="340"
13+
formats="QR_CODE, EAN_8, EAN_13"
14+
processEveryNthFrame="10"
15+
(scanResult)="onBarcodeScanResult($event)">
16+
</MLKitBarcodeScanner>
1717

18-
<ListView [items]="barcodes" class="m-t-20">
19-
<ng-template let-item="item">
20-
<GridLayout columns="2*, 3*">
21-
<Label col="0" class="m-5" [text]="item.format" horizontalAlignment="left"></Label>
22-
<Label col="1" class="m-5" [text]="item.value" horizontalAlignment="left"></Label>
23-
</GridLayout>
24-
</ng-template>
25-
</ListView>
18+
<ListView row="2" [items]="barcodes" class="m-t-20">
19+
<ng-template let-item="item">
20+
<GridLayout columns="2*, 3*">
21+
<Label col="0" class="mlkit-result" textWrap="true" [text]="item.format"></Label>
22+
<Label col="1" class="mlkit-result" textWrap="true" [text]="item.value"></Label>
23+
</GridLayout>
24+
</ng-template>
25+
</ListView>
2626

27-
</StackLayout>
28-
</ScrollView>
27+
</GridLayout>

‎demo-ng/app/tabs/mlkit/barcodescanning/barcodescanning.component.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ export class BarcodeScanningComponent {
1313
}>;
1414

1515
onBarcodeScanResult(event): void {
16-
console.log(">>> onBarcodeScanResult");
1716
const result: MLKitScanBarcodesResult = event.value;
1817
this.barcodes = result.barcodes;
1918
console.log(">>> onBarcodeScanResult, this.barcodes: " + JSON.stringify(this.barcodes));

‎demo-ng/app/tabs/mlkit/facedetection/facedetection.component.html‎

Lines changed: 31 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -2,33 +2,34 @@
22
<Label class="action-bar-title" text="Face detection"></Label>
33
</ActionBar>
44

5-
<ScrollView>
6-
<StackLayout>
7-
8-
<Label text="The scanner has been configured to detect faces every 5th frame. You can tweak this in your usage of the plugin." textWrap="true"></Label>
9-
10-
<MLKitFaceDetection
11-
width="300"
12-
height="340"
13-
processEveryNthFrame="5"
14-
(scanResult)="onFaceDetectionResult($event)">
15-
</MLKitFaceDetection>
16-
17-
<Label [text]="mlKitAllOK" textWrap="true"></Label>
18-
19-
<Image [src]="mlKitLastMatchImg" *ngIf="mlKitLastMatchImg"></Image>
20-
21-
<!--<Image [src]="scannedImage" width="200" *ngIf="scannedImage"></Image>-->
22-
23-
<ListView [items]="faces" class="m-t-20">
24-
<ng-template let-item="item">
25-
<GridLayout columns="*, *, *">
26-
<Label col="0" class="m-5" [text]="item.smilingProbability | number" horizontalAlignment="left"></Label>
27-
<Label col="1" class="m-5" [text]="item.leftEyeOpenProbability | number" horizontalAlignment="left"></Label>
28-
<Label col="2" class="m-5" [text]="item.rightEyeOpenProbability | number" horizontalAlignment="left"></Label>
29-
</GridLayout>
30-
</ng-template>
31-
</ListView>
32-
33-
</StackLayout>
34-
</ScrollView>
5+
<GridLayout rows="auto, auto, auto, auto, *">
6+
7+
<Label row="0" text="The scanner has been configured to detect faces every 5th frame. You can tweak this in your usage of the plugin." textWrap="true"></Label>
8+
9+
<MLKitFaceDetection
10+
row="1"
11+
width="300"
12+
height="340"
13+
processEveryNthFrame="5"
14+
(scanResult)="onFaceDetectionResult($event)">
15+
</MLKitFaceDetection>
16+
17+
<Label row="2" [text]="mlKitAllOK" textWrap="true" class="m-t-10 c-purple"></Label>
18+
19+
<GridLayout row="3" columns="*, *, *" class="m-t-5">
20+
<Label col="0" class="mlkit-result font-weight-bold" textWrap="true" text="Smiling"></Label>
21+
<Label col="1" class="mlkit-result font-weight-bold" textWrap="true" text="Left 👁 open"></Label>
22+
<Label col="2" class="mlkit-result font-weight-bold" textWrap="true" text="Right 👁 open"></Label>
23+
</GridLayout>
24+
25+
<ListView row="4" [items]="faces">
26+
<ng-template let-item="item">
27+
<GridLayout columns="*, *, *">
28+
<Label col="0" class="mlkit-result" textWrap="true" [text]="item.smilingProbability | number"></Label>
29+
<Label col="1" class="mlkit-result" textWrap="true" [text]="item.leftEyeOpenProbability | number"></Label>
30+
<Label col="2" class="mlkit-result" textWrap="true" [text]="item.rightEyeOpenProbability | number"></Label>
31+
</GridLayout>
32+
</ng-template>
33+
</ListView>
34+
35+
</GridLayout>

‎demo-ng/app/tabs/mlkit/facedetection/facedetection.component.ts‎

Lines changed: 7 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -8,43 +8,26 @@ import { ImageSource } from "tns-core-modules/image-source";
88
templateUrl: "./facedetection.component.html",
99
})
1010
export class FaceDetectionComponent {
11+
// TODO create a type (in the plugin) for these Arrays
1112
faces: Array<{
12-
smilingProbability: number;
13-
leftEyeOpenProbability: number;
14-
rightEyeOpenProbability: number;
13+
smilingProbability?: number;
14+
leftEyeOpenProbability?: number;
15+
rightEyeOpenProbability?: number;
1516
}>;
1617

1718
mlKitAllOK: string;
18-
mlKitLastMatchImg: ImageSource;
1919

2020
onFaceDetectionResult(scanResult: any): any {
2121
const value: MLKitDetectFacesResult = scanResult.value;
2222
if (value.faces.length > 0) {
23+
this.faces = value.faces;
24+
2325
let allSmilingAndEyesOpen = true;
2426
value.faces.forEach(face => {
2527
allSmilingAndEyesOpen = allSmilingAndEyesOpen && face.smilingProbability && face.leftEyeOpenProbability && face.rightEyeOpenProbability &&
2628
face.smilingProbability > 0.7 && face.leftEyeOpenProbability > 0.7 && face.rightEyeOpenProbability > 0.7;
2729
});
28-
this.mlKitAllOK = `All smiling and eyes open? ${allSmilingAndEyesOpen ? 'Yes, screen grabbed:' : 'Nope. Sad.'}`;
29-
30-
// value.faces.map(face => {
31-
// face.smilingProbability = FaceDetectionComponent.getPercentage(face.smilingProbability);
32-
// face.leftEyeOpenProbability = FaceDetectionComponent.getPercentage(face.leftEyeOpenProbability);
33-
// face.rightEyeOpenProbability = FaceDetectionComponent.getPercentage(face.rightEyeOpenProbability);
34-
// });
35-
this.faces = value.faces;
36-
37-
if (allSmilingAndEyesOpen && value.imageSource) {
38-
this.mlKitLastMatchImg = value.imageSource;
39-
}
30+
this.mlKitAllOK = `All smiling and eyes open? ${allSmilingAndEyesOpen ? 'Yes!' : 'Nope'}`;
4031
}
4132
}
42-
43-
private static getPercentage(input: number): number {
44-
if (isNaN(input)) {
45-
return 0;
46-
}
47-
return Math.round(input * 100);
48-
}
49-
5033
}

‎demo-ng/app/tabs/mlkit/imagelabeling/imagelabeling.component.html‎

Lines changed: 16 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,23 +2,22 @@
22
<Label class="action-bar-title" text="Image labeling"></Label>
33
</ActionBar>
44

5-
<ScrollView>
6-
<StackLayout>
5+
<GridLayout rows="auto, *">
76

8-
<MLKitImageLabeling
9-
width="300"
10-
height="340"
11-
(scanResult)="onImageLabeledResult($event)">
12-
</MLKitImageLabeling>
7+
<MLKitImageLabeling
8+
row="0"
9+
width="300"
10+
height="340"
11+
(scanResult)="onImageLabeledResult($event)">
12+
</MLKitImageLabeling>
1313

14-
<ListView [items]="labels" class="m-t-20">
15-
<ng-template let-item="item">
16-
<GridLayout columns="2*, 3*">
17-
<Label col="0" class="m-5" [text]="item.text" horizontalAlignment="left"></Label>
18-
<Label col="1" class="m-5" [text]="item.confidence | number" horizontalAlignment="left"></Label>
19-
</GridLayout>
20-
</ng-template>
21-
</ListView>
14+
<ListView row="1" [items]="labels" class="m-t-20">
15+
<ng-template let-item="item">
16+
<GridLayout columns="3*, 2*">
17+
<Label col="0" class="mlkit-result" textWrap="true" [text]="item.text"></Label>
18+
<Label col="1" class="mlkit-result" textWrap="true" [text]="item.confidence | number"></Label>
19+
</GridLayout>
20+
</ng-template>
21+
</ListView>
2222

23-
</StackLayout>
24-
</ScrollView>
23+
</GridLayout>

‎demo-ng/app/tabs/mlkit/mlkit.component.html‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@
44

55
<StackLayout orientation="horizontal" horizontalAlignment="center">
66
<Button text="Cameraroll" (tap)="fromCameraroll()" class="button button-mlkit"></Button>
7-
<Button text="Camera picture" (tap)="fromCameraPicture()" class="button button-mlkit"></Button>
8-
<Button text="Camera feed" (tap)="fromCameraFeed()" class="button button-mlkit"></Button>
7+
<Button text="Cam picture" (tap)="fromCameraPicture()" class="button button-mlkit"></Button>
8+
<Button text="Cam feed" (tap)="fromCameraFeed()" class="button button-mlkit"></Button>
99
</StackLayout>
1010

1111
</StackLayout>

‎demo-ng/app/tabs/mlkit/mlkit.component.ts‎

Lines changed: 38 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,17 @@
1-
import { Component, NgZone } from "@angular/core";
1+
import { Component } from "@angular/core";
22
import { ImageSource } from "tns-core-modules/image-source";
33

44
import { BarcodeFormat, MLKitScanBarcodesResult } from "nativescript-plugin-firebase/mlkit/barcodescanning";
55
import { MLKitRecognizeTextResult } from "nativescript-plugin-firebase/mlkit/textrecognition";
6+
import { MLKitLandmarkRecognitionResult } from "nativescript-plugin-firebase/mlkit/landmarkrecognition";
67
import { MLKitDetectFacesResult } from "nativescript-plugin-firebase/mlkit/facedetection";
7-
import { Image } from "tns-core-modules/ui/image";
88
import { action } from "tns-core-modules/ui/dialogs";
99
import { ImageAsset } from "tns-core-modules/image-asset";
1010
import * as ImagePicker from "nativescript-imagepicker";
1111
import * as Camera from "nativescript-camera";
1212
import { RouterExtensions } from "nativescript-angular";
13+
import { isIOS } from "tns-core-modules/platform";
14+
import { MLKitImageLabelingResult } from "nativescript-plugin-firebase/mlkit/imagelabeling";
1315

1416
const firebase = require("nativescript-plugin-firebase");
1517

@@ -19,18 +21,16 @@ const firebase = require("nativescript-plugin-firebase");
1921
templateUrl: "./mlkit.component.html",
2022
})
2123
export class MLKitComponent {
22-
public scannedImage: Image;
2324

2425
private mlkitFeatures: Array<string> = [
2526
"Text recognition",
2627
"Barcode scanning",
2728
"Face detection",
2829
"Image labeling",
29-
"Landmark recognition"
30+
"Landmark recognition (cloud)"
3031
];
3132

32-
constructor(private zone: NgZone,
33-
private routerExtensions: RouterExtensions) {
33+
constructor(private routerExtensions: RouterExtensions) {
3434
}
3535

3636
fromCameraFeed(): void {
@@ -48,6 +48,13 @@ export class MLKitComponent {
4848
to = "/tabs/mlkit/facedetection";
4949
} else if (pickedItem === "Image labeling") {
5050
to = "/tabs/mlkit/imagelabeling";
51+
} else if (pickedItem === "Landmark recognition (cloud)") {
52+
alert({
53+
title: `Not available`,
54+
message: `Landmark recognition is currently cloud-only, so that would be a bit too taxing on your dataplan.`,
55+
okButtonText: "Gotcha!"
56+
});
57+
return;
5158
}
5259
if (to !== undefined) {
5360
this.routerExtensions.navigate([to],
@@ -64,9 +71,12 @@ export class MLKitComponent {
6471
}
6572

6673
fromCameraPicture(): void {
74+
if (!isIOS) {
75+
Camera.requestPermissions();
76+
}
6777
Camera.takePicture({
68-
width: 1000,
69-
height: 1000,
78+
width: 800,
79+
height: 800,
7080
keepAspectRatio: true,
7181
saveToGallery: false,
7282
cameraFacing: "rear"
@@ -91,8 +101,8 @@ export class MLKitComponent {
91101

92102
const selected = selection[0];
93103
console.log(">>> selected: " + selected);
94-
selected.options.height = 1000;
95-
selected.options.width = 1000;
104+
selected.options.height = 800;
105+
selected.options.width = 800;
96106
selected.options.keepAspectRatio = true;
97107
selected.getImageAsync((image: any, error: any) => {
98108
console.log(">>> error: " + error);
@@ -126,6 +136,8 @@ export class MLKitComponent {
126136
this.detectFaces(imageSource);
127137
} else if (pickedItem === "Image labeling") {
128138
this.labelImage(imageSource);
139+
} else if (pickedItem === "Landmark recognition (cloud)") {
140+
this.recognizeLandmark(imageSource);
129141
}
130142
});
131143
}
@@ -144,6 +156,20 @@ export class MLKitComponent {
144156
.catch(errorMessage => console.log("ML Kit error: " + errorMessage));
145157
}
146158

159+
private recognizeLandmark(imageSource: ImageSource): void {
160+
firebase.mlkit.landmarkrecognition.recognizeLandmark({
161+
image: imageSource
162+
}).then(
163+
(result: MLKitLandmarkRecognitionResult) => {
164+
alert({
165+
title: `Result`,
166+
message: JSON.stringify(result.landmarks),
167+
okButtonText: "OK"
168+
});
169+
})
170+
.catch(errorMessage => console.log("ML Kit error: " + errorMessage));
171+
}
172+
147173
private scanBarcode(imageSource: ImageSource): void {
148174
firebase.mlkit.barcodescanning.scanBarcodes({
149175
image: imageSource,
@@ -178,10 +204,10 @@ export class MLKitComponent {
178204
image: imageSource,
179205
confidenceThreshold: 0.3
180206
}).then(
181-
(result: MLKitDetectFacesResult) => {
207+
(result: MLKitImageLabelingResult) => {
182208
alert({
183209
title: `Result`,
184-
message: JSON.stringify(result.faces),
210+
message: JSON.stringify(result.labels),
185211
okButtonText: "OK"
186212
});
187213
})

0 commit comments

Comments
 (0)