Skip to content
This repository was archived by the owner on Apr 4, 2023. It is now read-only.

Commit 32d768a

Browse files
EddyVerbruggenEddyVerbruggen
committed
Add support for ML Kit Natural Language Translation #1287 
1 parent d8f67b2 commit 32d768a

File tree

13 files changed

+394
-10
lines changed

13 files changed

+394
-10
lines changed

‎demo-ng/app/tabs/mlkit/mlkit.component.ts‎

Lines changed: 66 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,9 @@ import { BarcodeFormat, MLKitScanBarcodesOnDeviceResult } from "nativescript-plu
66
import { MLKitCustomModelResult } from "nativescript-plugin-firebase/mlkit/custommodel";
77
import { MLKitDetectFacesOnDeviceResult } from "nativescript-plugin-firebase/mlkit/facedetection";
88
import { MLKitImageLabelingCloudResult, MLKitImageLabelingOnDeviceResult } from "nativescript-plugin-firebase/mlkit/imagelabeling";
9-
import { MLKitObjectDetectionResult } from "nativescript-plugin-firebase/mlkit/objectdetection";
109
import { MLKitLandmarkRecognitionCloudResult } from "nativescript-plugin-firebase/mlkit/landmarkrecognition";
1110
import { MLKitNaturalLanguageIdentificationResult } from "nativescript-plugin-firebase/mlkit/naturallanguageidentification";
11+
import { MLKitObjectDetectionResult } from "nativescript-plugin-firebase/mlkit/objectdetection";
1212
import { MLKitSmartReplyConversationMessage } from "nativescript-plugin-firebase/mlkit/smartreply";
1313
import { MLKitRecognizeTextResult } from "nativescript-plugin-firebase/mlkit/textrecognition";
1414
import * as fileSystemModule from "tns-core-modules/file-system";
@@ -39,6 +39,7 @@ export class MLKitComponent {
3939
"Custom model",
4040
"Landmark recognition (cloud)",
4141
"Language identification",
42+
"Text translation (to EN)",
4243
"Smart reply"
4344
];
4445

@@ -96,9 +97,10 @@ export class MLKitComponent {
9697
fromAppFolder(): void {
9798
const folder = fileSystemModule.knownFolders.currentApp();
9899
const path = fileSystemModule.path.join(folder.path, "/images/puppy.jpg");
99-
const exists = fileSystemModule.File.exists(path);
100-
console.log(`Does it exist: ${exists}`);
101-
100+
if (!fileSystemModule.File.exists(path)) {
101+
console.log(`File doesn't exist at path: ${path}`);
102+
return;
103+
}
102104
const imageSource = fromFile(path);
103105
this.pickedImage = imageSource;
104106
// give the user some time to to see the picture
@@ -119,7 +121,7 @@ export class MLKitComponent {
119121
new ImageSource().fromAsset(imageAsset).then(imageSource => {
120122
this.pickedImage = imageSource;
121123
// give the user some time to to see the picture
122-
setTimeout(() => this.selectMLKitFeature(imageSource), 500);
124+
setTimeout(() => this.selectMLKitFeature(imageSource), 700);
123125
});
124126
});
125127
}
@@ -199,6 +201,8 @@ export class MLKitComponent {
199201
this.customModel(imageSource);
200202
} else if (pickedItem === "Language identification") {
201203
this.languageIdentification(imageSource);
204+
} else if (pickedItem === "Text translation (to EN)") {
205+
this.textTranslation(imageSource);
202206
} else if (pickedItem === "Smart reply") {
203207
this.smartReply(imageSource);
204208
}
@@ -267,6 +271,63 @@ export class MLKitComponent {
267271
}).catch(errorMessage => console.log("ML Kit error: " + errorMessage));
268272
}
269273

274+
private textTranslation(imageSource: ImageSource): void {
275+
// First recognize text, then get its language, then translate the text
276+
firebase.mlkit.textrecognition.recognizeTextOnDevice({
277+
image: imageSource
278+
}).then((result: MLKitRecognizeTextResult) => {
279+
if (!result || !result.text) {
280+
alert({
281+
title: `No text found`,
282+
message: `MLKit text recognition could not find any text in your image - please try a different one`,
283+
okButtonText: "Damn! OK.."
284+
});
285+
return;
286+
}
287+
288+
firebase.mlkit.naturallanguageidentification.identifyNaturalLanguage({
289+
text: result.text
290+
}).then((languageIdResult: MLKitNaturalLanguageIdentificationResult) => {
291+
292+
if (!languageIdResult || !languageIdResult.languageCode) {
293+
alert({
294+
title: `Unknown language`,
295+
message: `Could not determine language, please try again`,
296+
okButtonText: "Damn! OK.."
297+
});
298+
299+
} else if (languageIdResult.languageCode.toLowerCase() === "en") {
300+
alert({
301+
title: `Dude..`,
302+
message: `Language already English.. no need to translate 😉`,
303+
okButtonText: "Lol, fine.."
304+
});
305+
306+
} else {
307+
// translate to EN
308+
console.log(`Translating '${languageIdResult.languageCode}' to 'en'`);
309+
310+
firebase.mlkit.translation.ensureTranslationModelDownloaded({
311+
from: "" + languageIdResult.languageCode,
312+
to: "en"
313+
}).then(() => {
314+
firebase.mlkit.translation.translateText({
315+
from: languageIdResult.languageCode,
316+
to: "en",
317+
text: result.text
318+
}).then(result => {
319+
alert({
320+
title: "Translated to English:",
321+
message: result,
322+
okButtonText: "Awesome!"
323+
});
324+
}).catch(console.error)
325+
}).catch(console.error);
326+
}
327+
}).catch(errorMessage => console.log("ML Kit error: " + errorMessage));
328+
}).catch(errorMessage => console.log("ML Kit error: " + errorMessage));
329+
}
330+
270331
// it would be easier to hardcode the conversation, but this fits better with the other image-based examples
271332
private smartReply(imageSource: ImageSource): void {
272333
firebase.mlkit.textrecognition.recognizeTextOnDevice({

‎demo-ng/firebase.nativescript.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,5 +25,6 @@
2525
"ml_kit_object_detection": true,
2626
"ml_kit_custom_model": true,
2727
"ml_kit_natural_language_identification": true,
28+
"ml_kit_natural_language_translation": true,
2829
"ml_kit_natural_language_smartreply": true
2930
}

‎publish/scripts/installer.js‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,10 @@ function promptQuestions() {
242242
name: 'ml_kit_natural_language_identification',
243243
description: 'With Ml Kit, do you want to recognize natural languages? (y/n)',
244244
default: 'n'
245+
}, {
246+
name: 'ml_kit_natural_language_translation',
247+
description: 'With Ml Kit, do you want to translate text? (y/n)',
248+
default: 'n'
245249
}, {
246250
name: 'ml_kit_natural_language_smartreply',
247251
description: 'With Ml Kit, do you want to use smart reply? (y/n)',
@@ -414,7 +418,8 @@ end`) + `
414418
` + (isSelected(result.ml_kit) && isSelected(result.ml_kit_custom_model) ? `` : `#`) + `pod 'Firebase/MLModelInterpreter'
415419
` + (isSelected(result.ml_kit) && isSelected(result.ml_kit_natural_language_identification) ? `` : `#`) + `pod 'Firebase/MLNaturalLanguage'
416420
` + (isSelected(result.ml_kit) && isSelected(result.ml_kit_natural_language_identification) ? `` : `#`) + `pod 'Firebase/MLNLLanguageID'
417-
` + (isSelected(result.ml_kit) && isSelected(result.ml_kit_natural_language_smartreply) ? `` : `#`) + `pod 'Firebase/MLCommon'
421+
` + (isSelected(result.ml_kit) && isSelected(result.ml_kit_natural_language_translation) ? `` : `#`) + `pod 'Firebase/MLNLTranslate'
422+
` + (isSelected(result.ml_kit) && (isSelected(result.ml_kit_natural_language_smartreply) || isSelected(result.ml_kit_natural_language_translation)) ? `` : `#`) + `pod 'Firebase/MLCommon'
418423
` + (isSelected(result.ml_kit) && isSelected(result.ml_kit_natural_language_smartreply) ? `` : `#`) + `pod 'Firebase/MLNLSmartReply'
419424
420425
# Facebook Authentication
@@ -769,8 +774,9 @@ dependencies {
769774
` + (isSelected(result.ml_kit_image_labeling) ? `` : `//`) + ` implementation "com.google.firebase:firebase-ml-vision-image-label-model:18.0.0"
770775
` + (isSelected(result.ml_kit_object_detection) ? `` : `//`) + ` implementation "com.google.firebase:firebase-ml-vision-object-detection-model:19.0.1"
771776
` + (isSelected(result.ml_kit_custom_model) ? `` : `//`) + ` implementation "com.google.firebase:firebase-ml-model-interpreter:21.0.0"
772-
` + (isSelected(result.ml_kit_natural_language_identification) || isSelected(result.ml_kit_natural_language_smartreply) ? `` : `//`) + ` implementation "com.google.firebase:firebase-ml-natural-language:21.0.2"
777+
` + (isSelected(result.ml_kit_natural_language_identification) || isSelected(result.ml_kit_natural_language_smartreply) || isSelected(result.ml_kit_natural_language_translation) ? `` : `//`) + ` implementation "com.google.firebase:firebase-ml-natural-language:21.0.2"
773778
` + (isSelected(result.ml_kit_natural_language_identification) ? `` : `//`) + ` implementation "com.google.firebase:firebase-ml-natural-language-language-id-model:20.0.5"
779+
` + (isSelected(result.ml_kit_natural_language_translation) ? `` : `//`) + ` implementation "com.google.firebase:firebase-ml-natural-language-translate-model:20.0.5"
774780
` + (isSelected(result.ml_kit_natural_language_smartreply) ? `` : `//`) + ` implementation "com.google.firebase:firebase-ml-natural-language-smart-reply-model:20.0.5"
775781
776782
// Facebook Authentication

‎src/mlkit/index.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import * as objectdetection from "./objectdetection";
66
import * as landmarkrecognition from "./landmarkrecognition";
77
import * as custommodel from "./custommodel";
88
import * as naturallanguageidentification from "./naturallanguageidentification";
9+
import * as translation from "./translation";
910
import * as smartreply from "./smartreply";
1011

1112
import { ImageSource } from "tns-core-modules/image-source";
@@ -48,6 +49,7 @@ export {
4849
landmarkrecognition,
4950
custommodel,
5051
naturallanguageidentification,
52+
translation,
5153
smartreply
5254
};
5355

‎src/mlkit/naturallanguageidentification/index.android.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@ export function identifyNaturalLanguage(options: MLKitNaturalLanguageIdentificat
2121
languageIdentifier.identifyLanguage(options.text)
2222
.addOnSuccessListener(new (<any>com.google.android.gms).tasks.OnSuccessListener({
2323
onSuccess: languageCode => {
24+
console.log("languageCode: " + languageCode);
25+
console.log(languageCode);
26+
console.log(typeof languageCode);
27+
console.log("languageCode.");
2428
if (languageCode && languageCode !== "und") {
2529
resolve({ languageCode })
2630
} else {

‎src/mlkit/translation/index.android.ts‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
import { MLKitTranslationModelDownloadOptions, MLKitTranslationOptions } from "./";
2+
3+
declare const com: any;
4+
5+
export function ensureTranslationModelDownloaded(options: MLKitTranslationModelDownloadOptions): Promise<void> {
6+
return new Promise<void>((resolve, reject) => {
7+
_downloadTranslationModelIfNeeded(options)
8+
.then(() => resolve())
9+
.catch(reject);
10+
});
11+
}
12+
13+
export function translateText(options: MLKitTranslationOptions): Promise<string> {
14+
return new Promise<string>((resolve, reject) => {
15+
try {
16+
const onSuccessListener = new com.google.android.gms.tasks.OnSuccessListener({
17+
onSuccess: (result: string) => resolve(result)
18+
});
19+
20+
const onFailureListener = new com.google.android.gms.tasks.OnFailureListener({
21+
onFailure: exception => reject(exception.getMessage())
22+
});
23+
24+
_downloadTranslationModelIfNeeded(options)
25+
.then(translator => {
26+
translator.translate(options.text)
27+
.addOnSuccessListener(onSuccessListener)
28+
.addOnFailureListener(onFailureListener);
29+
})
30+
.catch(reject);
31+
} catch (ex) {
32+
console.log("Error in firebase.mlkit.translation.translateText: " + ex);
33+
reject(ex);
34+
}
35+
});
36+
}
37+
38+
function _downloadTranslationModelIfNeeded(options: MLKitTranslationModelDownloadOptions): Promise<com.google.firebase.ml.naturallanguage.translate.FirebaseTranslator> {
39+
return new Promise<com.google.firebase.ml.naturallanguage.translate.FirebaseTranslator>((resolve, reject) => {
40+
try {
41+
const source = +com.google.firebase.ml.naturallanguage.translate.FirebaseTranslateLanguage.languageForLanguageCode(options.from);
42+
const target = +com.google.firebase.ml.naturallanguage.translate.FirebaseTranslateLanguage.languageForLanguageCode(options.to);
43+
44+
const firTranslatorOptions = new com.google.firebase.ml.naturallanguage.translate.FirebaseTranslatorOptions.Builder()
45+
.setSourceLanguage(source)
46+
.setTargetLanguage(target)
47+
.build();
48+
49+
const firTranslator = com.google.firebase.ml.naturallanguage.FirebaseNaturalLanguage.getInstance().getTranslator(firTranslatorOptions);
50+
51+
const modelDownloadConditions = new com.google.firebase.ml.common.modeldownload.FirebaseModelDownloadConditions.Builder()
52+
.requireWifi()
53+
.build();
54+
55+
const onSuccessListener = new com.google.android.gms.tasks.OnSuccessListener({
56+
onSuccess: () => resolve(firTranslator)
57+
});
58+
59+
const onFailureListener = new com.google.android.gms.tasks.OnFailureListener({
60+
onFailure: exception => reject(exception.getMessage())
61+
});
62+
63+
firTranslator.downloadModelIfNeeded(modelDownloadConditions)
64+
.addOnSuccessListener(onSuccessListener)
65+
.addOnFailureListener(onFailureListener);
66+
67+
} catch (ex) {
68+
console.log("Error downloading translation model: " + ex);
69+
reject(ex);
70+
}
71+
});
72+
}

‎src/mlkit/translation/index.d.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
export interface MLKitTranslationModelDownloadOptions {
2+
from: string;
3+
to: string;
4+
}
5+
6+
export interface MLKitTranslationOptions extends MLKitTranslationModelDownloadOptions {
7+
text: string;
8+
}
9+
10+
export declare function ensureTranslationModelDownloaded(options: MLKitTranslationModelDownloadOptions): Promise<void>;
11+
12+
export declare function translateText(options: MLKitTranslationOptions): Promise<Array<string>>;

‎src/mlkit/translation/index.ios.ts‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
import { MLKitTranslationModelDownloadOptions, MLKitTranslationOptions } from "./";
2+
3+
export function ensureTranslationModelDownloaded(options: MLKitTranslationModelDownloadOptions): Promise<void> {
4+
return new Promise<void>((resolve, reject) => {
5+
_downloadTranslationModelIfNeeded(options)
6+
.then(() => resolve())
7+
.catch(reject);
8+
});
9+
}
10+
11+
export function translateText(options: MLKitTranslationOptions): Promise<string> {
12+
return new Promise<string>((resolve, reject) => {
13+
try {
14+
_downloadTranslationModelIfNeeded(options)
15+
.then(firTranslator => {
16+
firTranslator.translateTextCompletion(options.text, ((result: string, error: NSError) => {
17+
error ? reject(error.localizedDescription) : resolve(result);
18+
}));
19+
})
20+
.catch(reject);
21+
} catch (ex) {
22+
console.log("Error in firebase.mlkit.translation.translateText: " + ex);
23+
reject(ex);
24+
}
25+
});
26+
}
27+
28+
function _downloadTranslationModelIfNeeded(options: MLKitTranslationModelDownloadOptions): Promise<FIRTranslator> {
29+
return new Promise<FIRTranslator>((resolve, reject) => {
30+
try {
31+
const firTranslatorOptions = FIRTranslatorOptions.alloc().initWithSourceLanguageTargetLanguage(FIRTranslateLanguageForLanguageCode(options.from), FIRTranslateLanguageForLanguageCode(options.to));
32+
const nl = FIRNaturalLanguage.naturalLanguage();
33+
34+
// this line currently (plugin v 10.0.0) errors: CONSOLE ERROR file:///node_modules/@angular/core/fesm5/core.js:26769:0: Error: -[FIRTranslateRemoteModel initWithName:allowsModelUpdates:initialConditions:updateConditions:isBaseModel:]: unrecognized selector sent to instance 0x280567360
35+
const firTranslator = nl.translatorWithOptions(firTranslatorOptions);
36+
37+
const firModelDownloadConditions = FIRModelDownloadConditions.alloc().initWithAllowsCellularAccessAllowsBackgroundDownloading(false, true);
38+
firTranslator.downloadModelIfNeededWithConditionsCompletion(firModelDownloadConditions, (error: NSError) => {
39+
error ? reject(error.localizedDescription) : resolve(firTranslator);
40+
});
41+
} catch (ex) {
42+
console.log("Error downloading translation model: " + ex);
43+
reject(ex);
44+
}
45+
});
46+
}

‎src/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@
6969
"development.setup": "npm run setup && npm link && cd ../demo && npm link nativescript-plugin-firebase && cd ../src",
7070
"android.list-devices": "tns device android --available-devices",
7171
"ios.list-devices": "tns device ios --available-devices",
72-
"generate.typings.ios": "echo '##### Make sure to first run demo-ng.ios #####' && cd ../demo-ng && TNS_DEBUG_METADATA_PATH=\"$(pwd)/metadata\" tns run ios && TNS_TYPESCRIPT_DECLARATIONS_PATH=\"$(pwd)/typings\" tns build ios && echo 'Now look for your library typings in demo-ng/typings!'",
72+
"generate.typings.ios": "echo '##### Make sure to first run demo-ng.ios.. if a build error occurs after metadata is generate but not typings, paste the typings command in the cmd line in the demo-ng folder. #####' && cd ../demo-ng && TNS_DEBUG_METADATA_PATH=\"$(pwd)/metadata\" tns build ios && TNS_TYPESCRIPT_DECLARATIONS_PATH=\"$(pwd)/typings\" tns build ios && echo 'Now look for your library typings in demo-ng/typings!'",
7373
"generate.typings.android": "echo '##### Download the latest relevant .aar file from https://mvnrepository.com/artifact/com.google.firebase, rename .aar to .zip and extract it. Then move classes.jar to the plugin /src folder' && java -jar ../../android-dts-generator/dts-generator/build/libs/dts-generator.jar -input ./classes.jar -output platforms/android/typings/$npm_package_config_typings && rimraf classes.jar && echo '##### Done! Deleted src/classes.jar, and generated typings in src/platforms/android/typings/'",
7474
"generate.typings.android.analytics": "npm run generate.typings.android --nativescript-plugin-firebase:typings=analytics-impl",
7575
"generate.typings.android.auth": "npm run generate.typings.android --nativescript-plugin-firebase:typings=auth",

0 commit comments

Comments
 (0)