Merge pull request #8 from dataiku/feature/add-jp-support

Add Japanese Support

Author: Muennighoff

CHANGELOG.md

@@ -1,4 +1,7 @@
 # Changelog
 
+## Version 1.3.4 - Feature release - 2021-12-31
+- Add Japanese support
+
 ## Version 1.3.3 - Patch release - 2021-09-08
 - Fixed gensism version to avoid api conflict with Flair package

code-env/python/spec/requirements.txt

@@ -3,11 +3,12 @@ flair==0.6.1
 gensim==3.8.0
 flask>=1.0,<1.1
 tqdm==4.50.0
-spacy==2.3.2
+spacy[ja]==2.3.2
 https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.3.1/en_core_web_sm-2.3.1.tar.gz
 https://github.com/explosion/spacy-models/releases/download/es_core_news_sm-2.3.1/es_core_news_sm-2.3.1.tar.gz
 https://github.com/explosion/spacy-models/releases/download/zh_core_web_sm-2.3.1/zh_core_web_sm-2.3.1.tar.gz
 https://github.com/explosion/spacy-models/releases/download/pl_core_news_sm-2.3.0/pl_core_news_sm-2.3.0.tar.gz
 https://github.com/explosion/spacy-models/releases/download/nb_core_news_sm-2.3.0/nb_core_news_sm-2.3.0.tar.gz
 https://github.com/explosion/spacy-models/releases/download/fr_core_news_sm-2.3.0/fr_core_news_sm-2.3.0.tar.gz
-https://github.com/explosion/spacy-models/releases/download/de_core_news_sm-2.3.0/de_core_news_sm-2.3.0.tar.gz
+https://github.com/explosion/spacy-models/releases/download/de_core_news_sm-2.3.0/de_core_news_sm-2.3.0.tar.gz
+https://github.com/explosion/spacy-models/releases/download/ja_core_news_sm-2.3.0/ja_core_news_sm-2.3.0.tar.gz

custom-recipes/named-entity-recognition-extract/recipe.json

@@ -75,6 +75,10 @@
                     "value": "de",
                     "label": "German"
                 },
+                {
+                    "value": "ja",
+                    "label": "Japanese"
+                },
                 {
                     "value": "nb",
                     "label": "Norwegian Bokmål"
@@ -142,4 +146,4 @@
             "defaultValue": false
         }
     ]
-}
+}

plugin.json

@@ -1,6 +1,6 @@
 {
     "id": "named-entity-recognition",
-    "version": "1.3.3",
+    "version": "1.3.4",
     "meta": {
         "label": "Named Entity Recognition",
         "category": "Natural Language Processing",

python-lib/ner_utils_spacy.py

@@ -11,6 +11,7 @@
     "pl": "nb_core_news_sm",
     "fr": "fr_core_news_sm",
     "de": "de_core_news_sm",
+    "ja": "ja_core_news_sm",
     "nb": "nb_core_news_sm",
 }
 

webapps/named-entity-recognition-spacy/body.html

@@ -29,6 +29,7 @@ <h2>Input Text</h3>
         <option selected value="en">English</option>
         <option value="fr">French</option>
         <option value="de">German</option>
+        <option value="ja">Japanese</option>
         <option value="nb">Norwegian Bokmål</option>
         <option value="pl">Polish</option>
         <option value="es">Spanish</option>