Skip to content
This repository was archived by the owner on Apr 4, 2023. It is now read-only.

Commit ff7faa2

Browse files
EddyVerbruggenEddyVerbruggen
committed
Add support for ML Kit's Natural Language and Language Identification #1127 
1 parent df77690 commit ff7faa2

File tree

8 files changed

+244
-9
lines changed

8 files changed

+244
-9
lines changed

‎demo-ng/app/tabs/mlkit/languageidentification/languageidentification.component.html‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
<ActionBar class="action-bar">
2+
<Label class="action-bar-title" text="Language identification"></Label>
3+
</ActionBar>
4+
5+
<GridLayout>
6+
<MLKitTextRecognition
7+
width="100%"
8+
height="100%"
9+
processEveryNthFrame="60"
10+
[torchOn]="torchOn"
11+
(scanResult)="onTextRecognitionResult($event)">
12+
</MLKitTextRecognition>
13+
14+
<GridLayout rows="*, 320, *" columns="*, 5/6*, *">
15+
<Label class="mask" row="0" col="0" colSpan="3"></Label>
16+
<Label class="mask" row="2" col="0" colSpan="3"></Label>
17+
<Label class="mask" row="1" col="0"></Label>
18+
<Label class="mask" row="1" col="2"></Label>
19+
<GridLayout row="1" col="1" rows="1/6*, *, 1/6*" columns="1/6*, *, 1/6*">
20+
<Label class="frame-top-left" row="0" col="0"></Label>
21+
<Label class="frame-top-right" row="0" col="2"></Label>
22+
<Label class="frame-bottom-left" row="2" col="0"></Label>
23+
<Label class="frame-bottom-right" row="2" col="2"></Label>
24+
<StackLayout class="swing" row="0" col="0" colSpan="3">
25+
<Label height="1" marginBottom="1" borderBottomWidth="1" borderColor="rgba(81, 184, 237, 0.1)"></Label>
26+
<Label height="1" marginBottom="1" borderBottomWidth="1" borderColor="rgba(81, 184, 237, 0.2)"></Label>
27+
<Label height="1" marginBottom="1" borderBottomWidth="1" borderColor="rgba(81, 184, 237, 0.3)"></Label>
28+
<Label height="1" marginBottom="1" borderBottomWidth="1" borderColor="rgba(81, 184, 237, 0.4)"></Label>
29+
<Label height="1" marginBottom="1" borderBottomWidth="1" borderColor="rgba(81, 184, 237, 0.5)"></Label>
30+
<Label height="1" marginBottom="1" borderBottomWidth="1" borderColor="rgba(81, 184, 237, 0.6)"></Label>
31+
<Label height="1" marginBottom="1" borderBottomWidth="1" borderColor="rgba(81, 184, 237, 0.7)"></Label>
32+
<Label height="1" marginBottom="1" borderBottomWidth="1" borderColor="rgba(81, 184, 237, 0.8)"></Label>
33+
<Label height="1" marginBottom="1" borderBottomWidth="1" borderColor="rgba(81, 184, 237, 0.9)"></Label>
34+
<Label height="1" marginBottom="1" borderBottomWidth="1" borderColor="rgba(81, 184, 237, 1)"></Label>
35+
</StackLayout>
36+
</GridLayout>
37+
38+
<ListView separatorColor="transparent" row="0" rowSpan="3" col="0" colSpan="3" [items]="possibleLanguages" class="m-t-20" backgroundColor="transparent">
39+
<ng-template let-item="item">
40+
<StackLayout orientation="horizontal">
41+
<Label class="mlkit-result" textWrap="true" [text]="item.languageCode"></Label>
42+
<Label class="mlkit-result" textWrap="true" [text]="item.confidence"></Label>
43+
</StackLayout>
44+
</ng-template>
45+
</ListView>
46+
</GridLayout>
47+
48+
<GridLayout rows="auto" columns="auto, auto" horizontalAlignment="right" class="m-t-4 m-r-8">
49+
<Label col="0" text="Torch" class="c-white" [class.disabled]="!torchOn"></Label>
50+
<Switch col="1" [checked]="torchOn" (checkedChange)="toggleTorch($event)"></Switch>
51+
</GridLayout>
52+
53+
</GridLayout>

‎demo-ng/app/tabs/mlkit/languageidentification/languageidentification.component.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
import { Component } from "@angular/core";
2+
import {
3+
indentifyPossibleLanguages,
4+
MLKitNaturalLanguageIdentificationLanguage
5+
} from "nativescript-plugin-firebase/mlkit/naturallanguageidentification";
6+
import { MLKitRecognizeTextResult } from "nativescript-plugin-firebase/mlkit/textrecognition";
7+
import { AbstractMLKitViewComponent } from "~/tabs/mlkit/abstract.mlkitview.component";
8+
9+
@Component({
10+
selector: "mlkit-languageidentification",
11+
moduleId: module.id,
12+
templateUrl: "./languageidentification.component.html",
13+
})
14+
export class LanguageIdentificationComponent extends AbstractMLKitViewComponent {
15+
possibleLanguages: Array<MLKitNaturalLanguageIdentificationLanguage>;
16+
17+
onTextRecognitionResult(scanResult: any): void {
18+
const result: MLKitRecognizeTextResult = scanResult.value;
19+
if (!result.text) {
20+
return;
21+
}
22+
23+
indentifyPossibleLanguages({
24+
text: result.text,
25+
confidenceThreshold: 0.05
26+
}).then(
27+
(languageIdResults: Array<MLKitNaturalLanguageIdentificationLanguage>) => {
28+
console.log(">> results: " + JSON.stringify(languageIdResults));
29+
this.possibleLanguages = languageIdResults
30+
})
31+
.catch(errorMessage => console.log("ML Kit error: " + errorMessage));
32+
}
33+
}

‎demo-ng/app/tabs/mlkit/mlkit.component.ts‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import {
1010
MLKitImageLabelingOnDeviceResult
1111
} from "nativescript-plugin-firebase/mlkit/imagelabeling";
1212
import { MLKitLandmarkRecognitionCloudResult } from "nativescript-plugin-firebase/mlkit/landmarkrecognition";
13+
import { MLKitNaturalLanguageIdentificationResult } from "nativescript-plugin-firebase/mlkit/naturallanguageidentification";
1314
import { MLKitRecognizeTextResult } from "nativescript-plugin-firebase/mlkit/textrecognition";
1415
import * as fileSystemModule from "tns-core-modules/file-system";
1516
import { ImageAsset } from "tns-core-modules/image-asset";
@@ -36,15 +37,17 @@ export class MLKitComponent {
3637
"Image labeling (on device)",
3738
"Image labeling (cloud)",
3839
"Custom model",
39-
"Landmark recognition (cloud)"
40+
"Landmark recognition (cloud)",
41+
"Language identification (on device)"
4042
];
4143

4244
private mlkitOnDeviceFeatures: Array<string> = [
4345
"Text recognition",
4446
"Barcode scanning",
4547
"Face detection",
4648
"Image labeling",
47-
"Custom model"
49+
"Custom model",
50+
"Language identification"
4851
];
4952

5053
constructor(private routerExtensions: RouterExtensions,
@@ -68,6 +71,8 @@ export class MLKitComponent {
6871
to = "/tabs/mlkit/imagelabeling";
6972
} else if (pickedItem === "Custom model") {
7073
to = "/tabs/mlkit/custommodel";
74+
} else if (pickedItem === "Language identification") {
75+
to = "/tabs/mlkit/languageidentification";
7176
}
7277

7378
if (to !== undefined) {
@@ -186,6 +191,8 @@ export class MLKitComponent {
186191
this.recognizeLandmarkCloud(imageSource);
187192
} else if (pickedItem === "Custom model") {
188193
this.customModel(imageSource);
194+
} else if (pickedItem === "Language identification (on device)") {
195+
this.languageIdentification(imageSource);
189196
}
190197
});
191198
}
@@ -235,6 +242,27 @@ export class MLKitComponent {
235242
.catch(errorMessage => console.log("ML Kit error: " + errorMessage));
236243
}
237244

245+
private languageIdentification(imageSource: ImageSource): void {
246+
// First recognize text, then get its language
247+
firebase.mlkit.textrecognition.recognizeTextOnDevice({
248+
image: imageSource
249+
}).then(
250+
(result: MLKitRecognizeTextResult) => {
251+
firebase.mlkit.naturallanguageidentification.identifyNaturalLanguage({
252+
text: result.text
253+
}).then(
254+
(languageIdResult: MLKitNaturalLanguageIdentificationResult) => {
255+
alert({
256+
title: `Result`,
257+
message: `Language code: ${languageIdResult.languageCode}`,
258+
okButtonText: "OK"
259+
});
260+
})
261+
.catch(errorMessage => console.log("ML Kit error: " + errorMessage));
262+
})
263+
.catch(errorMessage => console.log("ML Kit error: " + errorMessage));
264+
}
265+
238266
private customModel(imageSource: ImageSource): void {
239267
firebase.mlkit.custommodel.useCustomModel({
240268
image: imageSource,

‎demo-ng/app/tabs/tabs-routing.module.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,16 @@ import { BarcodeScanningComponent } from "~/tabs/mlkit/barcodescanning/barcodesc
88
import { FaceDetectionComponent } from "~/tabs/mlkit/facedetection/facedetection.component";
99
import { ImageLabelingComponent } from "~/tabs/mlkit/imagelabeling/imagelabeling.component";
1010
import { CustomModelComponent } from "~/tabs/mlkit/custommodel/custommodel.component";
11+
import { LanguageIdentificationComponent } from "~/tabs/mlkit/languageidentification/languageidentification.component";
1112

1213
const routes: Routes = [
1314
{ path: "", component: TabsComponent },
1415
{ path: "mlkit/textrecognition", component: TextRecognitionComponent },
1516
{ path: "mlkit/barcodescanning", component: BarcodeScanningComponent },
1617
{ path: "mlkit/facedetection", component: FaceDetectionComponent },
1718
{ path: "mlkit/imagelabeling", component: ImageLabelingComponent },
18-
{ path: "mlkit/custommodel", component: CustomModelComponent }
19+
{ path: "mlkit/custommodel", component: CustomModelComponent },
20+
{ path: "mlkit/languageidentification", component: LanguageIdentificationComponent }
1921
];
2022

2123
@NgModule({

‎demo-ng/app/tabs/tabs.module.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import { BarcodeScanningComponent } from "~/tabs/mlkit/barcodescanning/barcodesc
1111
import { FaceDetectionComponent } from "~/tabs/mlkit/facedetection/facedetection.component";
1212
import { ImageLabelingComponent } from "~/tabs/mlkit/imagelabeling/imagelabeling.component";
1313
import { CustomModelComponent } from "~/tabs/mlkit/custommodel/custommodel.component";
14+
import { LanguageIdentificationComponent } from "~/tabs/mlkit/languageidentification/languageidentification.component";
1415

1516
import { registerElement } from "nativescript-angular/element-registry";
1617
registerElement("MLKitBarcodeScanner", () => require("nativescript-plugin-firebase/mlkit/barcodescanning").MLKitBarcodeScanner);
@@ -32,7 +33,8 @@ registerElement("MLKitCustomModel", () => require("nativescript-plugin-firebase/
3233
MLKitComponent,
3334
TabsComponent,
3435
TextRecognitionComponent,
35-
CustomModelComponent
36+
CustomModelComponent,
37+
LanguageIdentificationComponent
3638
],
3739
schemas: [
3840
NO_ERRORS_SCHEMA

‎src/mlkit/naturallanguageidentification/index.android.ts‎

Lines changed: 59 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,69 @@
1-
import { MLKitNaturalLanguageIdentificationOptions, MLKitNaturalLanguageIdentificationResult } from "./index";
1+
import {
2+
MLKitNaturalLanguageIdentificationLanguage,
3+
MLKitNaturalLanguageIdentificationOptions,
4+
MLKitNaturalLanguageIdentificationResult
5+
} from "./index";
26

37
export function identifyNaturalLanguage(options: MLKitNaturalLanguageIdentificationOptions): Promise<MLKitNaturalLanguageIdentificationResult> {
48
return new Promise((resolve, reject) => {
59
try {
6-
reject("Not implemented yet, because of build issues");
10+
const languageIdentifier: com.google.firebase.ml.naturallanguage.languageid.FirebaseLanguageIdentification =
11+
com.google.firebase.ml.naturallanguage.FirebaseNaturalLanguage.getInstance().getLanguageIdentification(
12+
new com.google.firebase.ml.naturallanguage.languageid.FirebaseLanguageIdentificationOptions.Builder()
13+
.setConfidenceThreshold(options.confidenceThreshold || 0.5)
14+
.build());
15+
16+
languageIdentifier.identifyLanguage(options.text)
17+
.addOnSuccessListener(new (<any>com.google.android.gms).tasks.OnSuccessListener({
18+
onSuccess: languageCode => {
19+
if (languageCode && languageCode !== "und") {
20+
resolve({ languageCode })
21+
} else {
22+
resolve();
23+
}
24+
}
25+
}))
26+
.addOnFailureListener(new (<any>com.google.android.gms).tasks.OnFailureListener({
27+
onFailure: exception => reject(exception.getMessage())
28+
}));
729
} catch (ex) {
830
console.log("Error in firebase.mlkit.identifyNaturalLanguage: " + ex);
931
reject(ex);
1032
}
1133
});
1234
}
35+
36+
export function indentifyPossibleLanguages(options: MLKitNaturalLanguageIdentificationOptions): Promise<Array<MLKitNaturalLanguageIdentificationLanguage>> {
37+
return new Promise((resolve, reject) => {
38+
try {
39+
const languageIdentifier: com.google.firebase.ml.naturallanguage.languageid.FirebaseLanguageIdentification =
40+
com.google.firebase.ml.naturallanguage.FirebaseNaturalLanguage.getInstance().getLanguageIdentification(
41+
new com.google.firebase.ml.naturallanguage.languageid.FirebaseLanguageIdentificationOptions.Builder()
42+
.setConfidenceThreshold(options.confidenceThreshold || 0.01)
43+
.build());
44+
45+
languageIdentifier.identifyPossibleLanguages(options.text)
46+
.addOnSuccessListener(new (<any>com.google.android.gms).tasks.OnSuccessListener({
47+
onSuccess: languages => {
48+
const langs: Array<MLKitNaturalLanguageIdentificationLanguage> = [];
49+
if (languages && languages.get(0).getLanguageCode() !== "und") { // und = undetermined
50+
for (let i = 0; i < languages.size(); i++) {
51+
const l = languages.get(i);
52+
langs.push({
53+
languageCode: l.getLanguageCode(),
54+
confidence: l.getConfidence()
55+
})
56+
}
57+
}
58+
resolve(langs);
59+
}
60+
}))
61+
.addOnFailureListener(new (<any>com.google.android.gms).tasks.OnFailureListener({
62+
onFailure: exception => reject(exception.getMessage())
63+
}));
64+
} catch (ex) {
65+
console.log("Error in firebase.mlkit.indentifyPossibleLanguages: " + ex);
66+
reject(ex);
67+
}
68+
});
69+
}

‎src/mlkit/naturallanguageidentification/index.d.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,18 @@ export interface MLKitNaturalLanguageIdentificationLanguage {
44
}
55

66
export interface MLKitNaturalLanguageIdentificationResult {
7-
landmarks: Array<MLKitNaturalLanguageIdentificationLanguage>;
7+
languageCode?: string;
88
}
99

1010
export interface MLKitNaturalLanguageIdentificationOptions {
11+
text: string;
12+
/**
13+
* By default, ML Kit returns a non-undefined value only when it identifies the language with a confidence value higher than X.
14+
* X is 0.5 for 'identifyNaturalLanguage'.
15+
* X is 0.01 for 'indentifyPossibleLanguages'.
16+
*/
17+
confidenceThreshold?: number;
1118
}
1219

1320
export declare function identifyNaturalLanguage(options: MLKitNaturalLanguageIdentificationOptions): Promise<MLKitNaturalLanguageIdentificationResult>;
21+
export declare function indentifyPossibleLanguages(options: MLKitNaturalLanguageIdentificationOptions): Promise<Array<MLKitNaturalLanguageIdentificationLanguage>>;

‎src/mlkit/naturallanguageidentification/index.ios.ts‎

Lines changed: 54 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,64 @@
1-
import { MLKitNaturalLanguageIdentificationOptions, MLKitNaturalLanguageIdentificationResult } from "./index";
1+
import {
2+
MLKitNaturalLanguageIdentificationLanguage,
3+
MLKitNaturalLanguageIdentificationOptions,
4+
MLKitNaturalLanguageIdentificationResult
5+
} from "./index";
26

37
export function identifyNaturalLanguage(options: MLKitNaturalLanguageIdentificationOptions): Promise<MLKitNaturalLanguageIdentificationResult> {
48
return new Promise((resolve, reject) => {
59
try {
6-
reject("Not implemented yet, because of build issues");
10+
const naturalLanguage = FIRNaturalLanguage.naturalLanguage();
11+
const languageId = naturalLanguage.languageIdentificationWithOptions(
12+
FIRLanguageIdentificationOptions.alloc().initWithConfidenceThreshold(options.confidenceThreshold || 0.5));
13+
14+
languageId.identifyLanguageForTextCompletion(options.text, (languageCode: string, error: NSError) => {
15+
if (error !== null) {
16+
console.log("Failed with error: " + error.localizedDescription);
17+
reject(error.localizedDescription);
18+
} else if (languageCode !== null && languageCode !== "und") { // und = undetermined
19+
console.log("Identified language: " + languageCode);
20+
resolve({languageCode});
21+
} else {
22+
console.log("No language was identified");
23+
resolve();
24+
}
25+
});
726
} catch (ex) {
827
console.log("Error in firebase.mlkit.identifyNaturalLanguage: " + ex);
928
reject(ex);
1029
}
1130
});
1231
}
32+
33+
export function indentifyPossibleLanguages(options: MLKitNaturalLanguageIdentificationOptions): Promise<Array<MLKitNaturalLanguageIdentificationLanguage>> {
34+
return new Promise((resolve, reject) => {
35+
try {
36+
const naturalLanguage = FIRNaturalLanguage.naturalLanguage();
37+
const languageId = naturalLanguage.languageIdentificationWithOptions(
38+
FIRLanguageIdentificationOptions.alloc().initWithConfidenceThreshold(options.confidenceThreshold || 0.01));
39+
40+
languageId.identifyPossibleLanguagesForTextCompletion(options.text, (languages: NSArray<FIRIdentifiedLanguage>, error: NSError) => {
41+
if (error !== null) {
42+
console.log("Failed with error: " + error.localizedDescription);
43+
reject(error.localizedDescription);
44+
} else if (languages.count === 1 && languages.objectAtIndex(0).languageCode === "und") { // und = undetermined
45+
console.log("No language was identified");
46+
resolve([]);
47+
} else {
48+
const langs: Array<MLKitNaturalLanguageIdentificationLanguage> = [];
49+
for (let i = 0; i < languages.count; i++) {
50+
const l = languages.objectAtIndex(i);
51+
langs.push({
52+
languageCode: l.languageCode,
53+
confidence: l.confidence
54+
})
55+
}
56+
resolve(langs);
57+
}
58+
});
59+
} catch (ex) {
60+
console.log("Error in firebase.mlkit.indentifyPossibleLanguages: " + ex);
61+
reject(ex);
62+
}
63+
});
64+
}

0 commit comments

Comments
 (0)