Merge pull request #19 from dataiku/feature/japanese-support

Japanese support

Author: Alexlandeau

README.md

@@ -2,7 +2,7 @@
 
 ![Build status](https://github.com/dataiku/dss-plugin-nlp-visualization/actions/workflows/auto-make.yml/badge.svg) ![GitHub release (latest by date)](https://img.shields.io/github/v/release/dataiku/dss-plugin-nlp-visualization?logo=github)  ![Support level](https://img.shields.io/badge/support-Unsupported-orange)
 
-This Dataiku DSS plugin provides a recipe to visualize text data in 58 languages using word clouds.
+This Dataiku DSS plugin provides a recipe to visualize text data in 59 languages using word clouds.
 
 Documentation: https://www.dataiku.com/product/plugins/nlp-visualization/
 

code-env/python/spec/requirements.txt

@@ -4,7 +4,7 @@ pymorphy2==0.9.1
 jieba==0.42.1
 pyvi==0.1
 regex==2020.11.13
-spacy[lookups,th]==2.3.5
+spacy[ja,lookups,th]==2.3.5
 emoji==1.2.0
 tqdm==4.50.2
 matplotlib==3.3.1

plugin.json

@@ -4,7 +4,7 @@
     "meta": {
         "label": "Text visualization",
         "category": "Natural Language Processing",
-        "description": "Visualize text data in 58 languages using word clouds",
+        "description": "Visualize text data in 59 languages using word clouds",
         "author": "Dataiku (Alex LANDEAU, Alex COMBESSIE)",
         "icon": "icon-quote-left",
         "tags": [
@@ -14,4 +14,4 @@
         "licenseInfo": "Apache Software License",
         "supportLevel": "NOT_SUPPORTED"
     }
-}
+}

python-lib/language_dict.py

@@ -28,6 +28,7 @@
     "id": "Indonesian",
     "is": "Icelandic",
     "it": "Italian",
+    "ja": "Japanese",
     "kn": "Kannada",
     "lb": "Luxembourgish",
     "lt": "Lithuanian",
@@ -64,7 +65,7 @@
 """dict: Languages supported by spaCy: https://spacy.io/usage/models#languages
 
 Dictionary with ISO 639-1 language code (key) and language name (value)
-Japanese and Korean were excluded for now because of system installation issues
+Korean is excluded for now because of system installation issues
 """
 
 SPACY_LANGUAGE_MODELS = {

python-lib/wordcloud_visualizer.py

@@ -65,6 +65,7 @@ class WordcloudVisualizer:
     """
     FONT_EXCEPTIONS_DICT = {
         "gu": "NotoSansMerged-Regular-2048upem.ttf",
+        "ja": "NotoSansCJKjp-Regular.otf",
         "kn": "NotoSansMerged-Regular-2048upem.ttf",
         "ml": "NotoSansMerged-Regular-2048upem.ttf",
         "te": "NotoSansMerged-Regular-2048upem.ttf",

resource/fonts/NotoSansCJKjp-Regular.otf

@@ -20,19 +20,28 @@ def test_tokenize_df_english():
     assert len(tokenized_document) == 15
 
 
+def test_tokenize_df_japanese():
+    input_df = pd.DataFrame({"input_text": ["期一会。 異体同心。 そうです。"]})
+    tokenizer = MultilingualTokenizer()
+    output_df = tokenizer.tokenize_df(df=input_df, text_column="input_text", language="ja")
+    tokenized_document = output_df[tokenizer.tokenized_column][0]
+    assert len(tokenized_document) == 9
+
+
 def test_tokenize_df_multilingual():
     input_df = pd.DataFrame(
         {
             "input_text": [
                 "I hope nothing. I fear nothing. I am free.",
                 " Les sanglots longs des violons d'automne",
                 "子曰：“學而不思則罔，思而不學則殆。”",
+                "期一会。 異体同心。 そうです。",
             ],
-            "language": ["en", "fr", "zh"],
+            "language": ["en", "fr", "zh", "ja"],
         }
     )
     tokenizer = MultilingualTokenizer(stopwords_folder_path=stopwords_folder_path)
     output_df = tokenizer.tokenize_df(df=input_df, text_column="input_text", language_column="language")
     tokenized_documents = output_df[tokenizer.tokenized_column]
     tokenized_documents_length = [len(doc) for doc in tokenized_documents]
-    assert tokenized_documents_length == [12, 8, 13]
+    assert tokenized_documents_length == [12, 8, 13, 9]