Skip to content
This repository was archived by the owner on Apr 4, 2023. It is now read-only.

Commit 751a5fc

Browse files
EddyVerbruggenEddyVerbruggen
committed
#699 Add ML Kit support
1 parent a79d05e commit 751a5fc

25 files changed

+434
-106
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ src/platforms/android/include.gradle
1717
!src/references.d.ts
1818
!src/analytics/analytics.d.ts
1919
!src/mlkit/*/index.d.ts
20+
!src/mlkit/mlkit-cameraview.d.ts
2021
!src/platforms/web/typings/firebase-webapi.d.ts
2122
!src/platforms/ios/typings/*.d.ts
2223
!src/platforms/android/typings/**/*.d.ts

‎demo-ng/app/app.css‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,4 +69,13 @@ button {
6969

7070
.button-invites {
7171
background-color: #1832d5;
72-
}
72+
}
73+
74+
.mlKitCamera {
75+
/*border-radius: 130;*/
76+
border-width: 8;
77+
border-color: #ddd;
78+
width: 260;
79+
height: 260;
80+
margin-top: 16;
81+
}

‎demo-ng/app/app.module.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@ import { ItemsComponent } from "./item/items.component";
88
import { registerElement } from "nativescript-angular/element-registry";
99

1010
registerElement("MLKitBarcodeScanner", () => require("nativescript-plugin-firebase/mlkit/barcodescanning").MLKitBarcodeScanner);
11-
// registerElement("MLKitFaceRecognition", () => require("nativescript-plugin-firebase/mlkit/facedetection").MLKitFaceRecognition);
12-
// registerElement("MLKitTextRecognition", () => require("nativescript-plugin-firebase/mlkit/textrecognition").MLKitTextRecognition);
11+
registerElement("MLKitFaceDetection", () => require("nativescript-plugin-firebase/mlkit/facedetection").MLKitFaceDetection);
12+
registerElement("MLKitTextRecognition", () => require("nativescript-plugin-firebase/mlkit/textrecognition").MLKitTextRecognition);
1313

1414
@NgModule({
1515
bootstrap: [

‎demo-ng/app/item/items.component.html‎

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,42 @@
55
<StackLayout class="page">
66
<Label text="After pressing those buttons, check the console log" textWrap="true"></Label>
77

8-
<MLKitBarcodeScanner
9-
width="400"
10-
height="300"
8+
<!--MLKitBarcodeScanner
9+
class="mlKitCamera"
10+
width="340"
11+
height="340"
1112
formats="QR_CODE, EAN_8"
1213
(scanResult)="onBarcodeScanResult($event)">
14+
</MLKitBarcodeScanner-->
15+
16+
<WrapLayout>
17+
<Button text="recognize text" (tap)="recognizeText()" class="button button-mlkit"></Button>
18+
<Button text="scan barcodes" (tap)="scanBarcode()" class="button button-mlkit"></Button>
19+
<Button text="detect faces" (tap)="detectFaces()" class="button button-mlkit"></Button>
20+
</WrapLayout>
21+
22+
<!--MLKitTextRecognition
23+
class="mlKitCamera"
24+
(scanResult)="onTextRecognitionResult($event)"
25+
(scanResultImage)="onScanResultImage($event)">
26+
</MLKitTextRecognition-->
27+
28+
<MLKitBarcodeScanner
29+
class="mlKitCamera"
30+
formats="QR_CODE, EAN_8, EAN_13"
31+
processEveryNthFrame="3"
32+
(scanResult)="onBarcodeScanResult($event)"
33+
(scanResultImage)="onScanResultImage($event)">
1334
</MLKitBarcodeScanner>
1435

36+
<Label [text]="mlKitTextValue" textWrap="true"></Label>
37+
38+
<Label [text]="mlKitAllOK" textWrap="true"></Label>
39+
40+
<Image [src]="mlKitLastMatchImg" *ngIf="mlKitLastMatchImg"></Image>
41+
42+
<Image [src]="scannedImage" width="200" *ngIf="scannedImage"></Image>
43+
1544
<Label text="Authentication" class="h2"></Label>
1645
<Button text="login anonymously" (tap)="loginAnonymously()" class="button button-user"></Button>
1746

‎demo-ng/app/item/items.component.ts‎

Lines changed: 132 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,15 @@
11
import { Component, NgZone } from "@angular/core";
22
import { firestore } from "nativescript-plugin-firebase";
33
import { Observable } from "rxjs/Observable";
4-
import { City } from "../model/City";
5-
import { MLKitScanBarcodesResult } from "../../../src/mlkit/barcodescanning";
4+
import { City } from "~/model/City";
5+
import { ImageSource } from "tns-core-modules/image-source";
66

7-
const firebase = require("nativescript-plugin-firebase/app");
7+
import { BarcodeFormat, MLKitScanBarcodesResult } from "nativescript-plugin-firebase/mlkit/barcodescanning";
8+
import { MLKitRecognizeTextResult } from "nativescript-plugin-firebase/mlkit/textrecognition";
9+
import { MLKitDetectFacesResult } from "nativescript-plugin-firebase/mlkit/facedetection";
10+
import { Image } from "tns-core-modules/ui/image";
11+
12+
const firebase = require("nativescript-plugin-firebase");
813
const firebaseWebApi = require("nativescript-plugin-firebase/app");
914

1015
// import { AngularFireModule } from 'angularfire2';
@@ -20,34 +25,142 @@ export class ItemsComponent {
2025

2126
public myCity$: Observable<City>;
2227
public myCities$: Observable<Array<City>>;
28+
public mlKitTextValue: string;
29+
public mlKitAllOK: string;
30+
public mlKitLastMatchImg: ImageSource;
31+
32+
public scannedImage: Image;
33+
2334
private city: City;
2435
private cities: Array<City> = [];
2536

2637
constructor(private zone: NgZone) {
2738
// AngularFireModule.initializeApp({});
2839
}
2940

41+
recognizeText(): void {
42+
const path = "~/images/please_walk_on_the_grass.jpg";
43+
const img = new ImageSource();
44+
img.fromFile(path)
45+
.then(() => {
46+
firebase.mlkit.textrecognition.recognizeText({
47+
image: img
48+
}).then(
49+
(result: MLKitRecognizeTextResult) => {
50+
alert({
51+
title: `Result from ${path}:`,
52+
message: JSON.stringify(result.features),
53+
okButtonText: "OK"
54+
});
55+
}, errorMessage => {
56+
console.log("ML Kit error: " + errorMessage);
57+
}
58+
);
59+
})
60+
.catch(err => console.log("Error in recognizeText: " + err));
61+
}
62+
63+
scanBarcode(): void {
64+
const path = "~/images/barcodes/2qrcodes.png";
65+
const img = new ImageSource();
66+
img.fromFile(path)
67+
.then(() => {
68+
firebase.mlkit.barcodescanning.scanBarcodes({
69+
image: img,
70+
formats: [BarcodeFormat.QR_CODE, BarcodeFormat.EAN_13]
71+
}).then(
72+
(result: MLKitScanBarcodesResult) => {
73+
alert({
74+
title: `Result from ${path}:`,
75+
message: JSON.stringify(result.barcodes),
76+
okButtonText: "OK"
77+
});
78+
}, errorMessage => {
79+
console.log("ML Kit error: " + errorMessage);
80+
}
81+
);
82+
})
83+
.catch(err => console.log("Error in scanBarcode: " + err));
84+
}
85+
86+
detectFaces(): void {
87+
const path = "~/images/faces.jpg";
88+
const img = new ImageSource();
89+
img.fromFile(path)
90+
.then(() => {
91+
firebase.mlkit.facedetection.detectFaces({
92+
image: img
93+
}).then(
94+
(result: MLKitDetectFacesResult) => {
95+
alert({
96+
title: `Result from ${path}:`,
97+
message: JSON.stringify(result.faces),
98+
okButtonText: "OK"
99+
});
100+
}, errorMessage => {
101+
console.log("ML Kit error: " + errorMessage);
102+
}
103+
);
104+
})
105+
.catch(err => console.log("Error in detectFaces: " + err));
106+
}
107+
30108
onBarcodeScanResult(event): void {
31109
const result: MLKitScanBarcodesResult = event.value;
32-
console.log("Received barcode(s): " + JSON.stringify(result));
110+
this.mlKitTextValue = JSON.stringify(result.barcodes);
111+
}
112+
113+
onScanResultImage(event): void {
114+
this.scannedImage = event.value;
115+
}
116+
117+
onTextRecognitionResult(scanResult: any): void {
118+
const value: MLKitRecognizeTextResult = scanResult.value;
119+
// this.mlKitTextValue = value.features.map(feature => feature.text).join("\n\n");
120+
this.mlKitTextValue = value.features.map(feature => JSON.stringify(feature)).join("\n\n");
121+
}
122+
123+
onFaceDetectionResult(scanResult: any): any {
124+
const value: MLKitDetectFacesResult = scanResult.value;
125+
if (value.faces.length > 0) {
126+
let allSmilingAndEyesOpen = true;
127+
value.faces.forEach(face => {
128+
allSmilingAndEyesOpen = allSmilingAndEyesOpen && face.smilingProbability && face.leftEyeOpenProbability && face.rightEyeOpenProbability &&
129+
face.smilingProbability > 0.7 && face.leftEyeOpenProbability > 0.7 && face.rightEyeOpenProbability > 0.7;
130+
});
131+
this.mlKitAllOK = `All smiling and eyes open? ${allSmilingAndEyesOpen ? 'Yes, screen grabbed:' : 'Nope. Sad.'}`;
132+
// model.set("textValue", value.faces.map(face => JSON.stringify(face)).join("\n"));
133+
this.mlKitTextValue = value.faces.map(face => `Smiling? ${this.round(face.smilingProbability)}%\nLeft eye open? ${this.round(face.leftEyeOpenProbability)}%\nRight eye open? ${this.round(face.rightEyeOpenProbability)}%`).join("\n\n");
134+
135+
if (allSmilingAndEyesOpen && value.imageSource) {
136+
this.mlKitLastMatchImg = value.imageSource;
137+
}
138+
}
139+
}
140+
141+
private round(input: number): number {
142+
if (isNaN(input)) {
143+
return 0;
144+
}
145+
return Math.round(input * 100);
33146
}
34147

35148
public loginAnonymously(): void {
36-
firebase.auth().signInAnonymously()
149+
firebaseWebApi.auth().signInAnonymously()
37150
.then(() => console.log("Logged in"))
38151
.catch(err => console.log("Login error: " + JSON.stringify(err)));
39152
}
40153

41154
public firestoreAdd(): void {
42-
firebase.firestore().collection("dogs").add({name: "Fido"})
155+
firebaseWebApi.firestore().collection("dogs").add({name: "Fido"})
43156
.then((docRef: firestore.DocumentReference) => {
44157
console.log("Fido added, ref: " + docRef.id);
45158
})
46159
.catch(err => console.log("Adding Fido failed, error: " + err));
47160
}
48161

49162
public firestoreSet(): void {
50-
firebase.firestore().collection("dogs").doc("fave")
163+
firebaseWebApi.firestore().collection("dogs").doc("fave")
51164
.set({name: "Woofie", last: "lastofwoofie", date: new Date()}, {merge: true})
52165
.then(() => {
53166
console.log("Woofie set");
@@ -56,7 +169,7 @@ export class ItemsComponent {
56169

57170

58171
// example from https://firebase.google.com/docs/firestore/query-data/get-data
59-
const citiesCollection = firebase.firestore().collection("cities");
172+
const citiesCollection = firebaseWebApi.firestore().collection("cities");
60173

61174
citiesCollection.doc("SF").set({
62175
name: "San Francisco",
@@ -108,7 +221,7 @@ export class ItemsComponent {
108221
}
109222

110223
public firestoreSetByAutoID(): void {
111-
firebase.firestore().collection("dogs").doc()
224+
firebaseWebApi.firestore().collection("dogs").doc()
112225
.set({name: "Woofie", last: "lastofwoofie", date: new Date()})
113226
.then(() => {
114227
console.log("Woofie set");
@@ -117,7 +230,7 @@ export class ItemsComponent {
117230
}
118231

119232
public firestoreUpdate(): void {
120-
firebase.firestore().collection("dogs").doc("fave")
233+
firebaseWebApi.firestore().collection("dogs").doc("fave")
121234
.update({name: "Woofieupdate", last: "updatedwoofie"})
122235
.then(() => {
123236
console.log("Woofie updated");
@@ -126,15 +239,15 @@ export class ItemsComponent {
126239
}
127240

128241
public firestoreGet(): void {
129-
const collectionRef: firestore.CollectionReference = firebase.firestore().collection("dogs");
242+
const collectionRef: firestore.CollectionReference = firebaseWebApi.firestore().collection("dogs");
130243
collectionRef.get()
131244
.then((querySnapshot: firestore.QuerySnapshot) => {
132245
querySnapshot.forEach(doc => console.log(`${doc.id} => ${JSON.stringify(doc.data())}`));
133246
})
134247
.catch(err => console.log("Get failed, error" + err));
135248

136249
// examples from https://firebase.google.com/docs/firestore/query-data/get-data
137-
const docRef: firestore.DocumentReference = firebase.firestore().collection("cities").doc("BJ");
250+
const docRef: firestore.DocumentReference = firebaseWebApi.firestore().collection("cities").doc("BJ");
138251

139252
docRef.get().then((doc: firestore.DocumentSnapshot) => {
140253
if (doc.exists) {
@@ -152,7 +265,7 @@ export class ItemsComponent {
152265

153266
public firestoreGetNested(): void {
154267
const mainStreetInSFDocRef: firestore.DocumentReference =
155-
firebase.firestore()
268+
firebaseWebApi.firestore()
156269
.collection("cities")
157270
.doc("SF")
158271
.collection("streets")
@@ -173,7 +286,7 @@ export class ItemsComponent {
173286

174287
firestoreDocumentObservable(): void {
175288
this.myCity$ = Observable.create(subscriber => {
176-
const docRef: firestore.DocumentReference = firebase.firestore().collection("cities").doc("SF");
289+
const docRef: firestore.DocumentReference = firebaseWebApi.firestore().collection("cities").doc("SF");
177290
docRef.onSnapshot((doc: firestore.DocumentSnapshot) => {
178291
this.zone.run(() => {
179292
this.city = <City>doc.data();
@@ -185,7 +298,7 @@ export class ItemsComponent {
185298

186299
firestoreCollectionObservable(): void {
187300
this.myCities$ = Observable.create(subscriber => {
188-
const colRef: firestore.CollectionReference = firebase.firestore().collection("cities");
301+
const colRef: firestore.CollectionReference = firebaseWebApi.firestore().collection("cities");
189302
colRef.onSnapshot((snapshot: firestore.QuerySnapshot) => {
190303
this.zone.run(() => {
191304
this.cities = [];
@@ -202,7 +315,7 @@ export class ItemsComponent {
202315
return;
203316
}
204317

205-
const docRef: firestore.DocumentReference = firebase.firestore().collection("cities").doc("SF");
318+
const docRef: firestore.DocumentReference = firebaseWebApi.firestore().collection("cities").doc("SF");
206319

207320
this.listenerUnsubscribe = docRef.onSnapshot((doc: firestore.DocumentSnapshot) => {
208321
if (doc.exists) {
@@ -224,7 +337,7 @@ export class ItemsComponent {
224337
}
225338

226339
public firestoreWhere(): void {
227-
const query: firestore.Query = firebase.firestore().collection("cities")
340+
const query: firestore.Query = firebaseWebApi.firestore().collection("cities")
228341
.where("state", "==", "CA")
229342
.where("population", "<", 550000);
230343

@@ -239,7 +352,7 @@ export class ItemsComponent {
239352
}
240353

241354
public firestoreWhereOrderLimit(): void {
242-
const query: firestore.Query = firebase.firestore().collection("cities")
355+
const query: firestore.Query = firebaseWebApi.firestore().collection("cities")
243356
.where("state", "==", "CA")
244357
.orderBy("population", "desc")
245358
.limit(2);
@@ -255,7 +368,7 @@ export class ItemsComponent {
255368
}
256369

257370
public firestoreDelete(): void {
258-
firebase.firestore().collection("dogs").doc("fave")
371+
firebaseWebApi.firestore().collection("dogs").doc("fave")
259372
.delete()
260373
.then(() => {
261374
console.log("Woofie deleted");

‎publish/scripts/installer.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -258,6 +258,7 @@ post_install do |installer|
258258
installer.pods_project.targets.each do |target|
259259
target.build_configurations.each do |config|
260260
config.build_settings['ENABLE_BITCODE'] = "NO"
261+
config.build_settings['CLANG_ALLOW_NON_MODULAR_INCLUDES_IN_FRAMEWORK_MODULES'] = "YES"
261262
end
262263
end
263264
end`) + `

‎src/mlkit/barcodescanning/barcodescanning-common.ts‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,6 @@ export const reportDuplicatesProperty = new Property<MLKitBarcodeScanner, boolea
4444
});
4545

4646
export abstract class MLKitBarcodeScanner extends MLKitCameraView {
47-
static scanResultEvent: string = "scanResult";
48-
4947
protected formats: string;
5048
protected preferFrontCamera: boolean;
5149
protected beepOnScan: boolean;

‎src/mlkit/barcodescanning/index.d.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ import { MLKitOptions } from "../";
22
import { BarcodeFormat } from "./barcodescanning-common";
33
import { MLKitResult, MLKitCameraView } from "../index";
44

5+
export { BarcodeFormat };
6+
57
export interface MLKitScanBarcodesResult extends MLKitResult {
68
barcodes: Array<{
79
value: string;

0 commit comments

Comments
 (0)