Merge pull request #7 from Alir3z4/kissarat-master

LANGUAGE_MAPPING

Thanks @kissarat @@dmiro

Author: Alir3z4

.gitignore

@@ -13,3 +13,6 @@ dist/
 logs/
 src/
 .c9/
+bin/
+develop-eggs/
+eggs/

.travis.yml

@@ -8,6 +8,7 @@ python:
 install:
     - git submodule init
     - git submodule update
+    - git submodule foreach git pull origin master
     - pip install -U setuptools coveralls
     - python bootstrap.py
     - ./bin/buildout

AUTHORS.rst

@@ -13,6 +13,7 @@ answer newbie questions, and generally made ``python-stop-words`` that much bett
 * Alireza Savand <alireza.savand@gmail.com>
 * Julien Fache <fantomas42@gmail.com>
 * David Miró <lite.3engine@gmail.com>
+* Taras Labiak <kissarat@gmail.com>
 
 
 A big THANK YOU goes to:

ChangeLog.rst

@@ -1,3 +1,11 @@
+2015.2.21
+=========
+----
+
+* Feature: ``LANGUAGE_MAPPING`` is loads from stop-words/languages.json
+* Fix: Made paths OS-independent
+
+
 2015.1.31
 =========
 ----

README.rst

@@ -67,7 +67,7 @@ Available languages
 * Spanish
 * Swedish
 * Turkish
-
+* Ukrainian
 
 Installation
 ------------

stop_words/__init__.py

@@ -1,29 +1,15 @@
+import json
 import os
 
-__VERSION__ = (2015, 1, 31)
+__VERSION__ = (2015, 2, 21)
 CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))
-STOP_WORDS_DIR = os.path.join(CURRENT_DIR, 'stop-words/')
+STOP_WORDS_DIR = os.path.join(CURRENT_DIR, 'stop-words')
 STOP_WORDS_CACHE = {}
 
-LANGUAGE_MAPPING = {
-    'ar': 'arabic',
-    'ca': 'catalan',
-    'da': 'danish',
-    'nl': 'dutch',
-    'en': 'english',
-    'fi': 'finnish',
-    'fr': 'french',
-    'de': 'german',
-    'hu': 'hungarian',
-    'it': 'italian',
-    'nb': 'norwegian',
-    'pt': 'portuguese',
-    'ro': 'romanian',
-    'ru': 'russian',
-    'es': 'spanish',
-    'sv': 'swedish',
-    'tr': 'turkish',
-}
+with open(os.path.join(STOP_WORDS_DIR, 'languages.json'), 'rb') as map_file:
+    buffer = map_file.read()
+    buffer = buffer.decode('ascii')
+    LANGUAGE_MAPPING = json.loads(buffer)
 
 AVAILABLE_LANGUAGES = LANGUAGE_MAPPING.values()
 
@@ -48,23 +34,25 @@ def get_stop_words(language):
     try:
         language = LANGUAGE_MAPPING[language]
     except KeyError:
-        pass
-
-    if language not in AVAILABLE_LANGUAGES:
-        raise StopWordError('"%s" language is unavailable.' % language)
+        if language not in AVAILABLE_LANGUAGES:
+            raise StopWordError('{0}" language is unavailable.'.format(
+                language
+            ))
 
     if language in STOP_WORDS_CACHE:
         return STOP_WORDS_CACHE[language]
 
+    language_filename = os.path.join(STOP_WORDS_DIR, language + '.txt')
     try:
-        language_filename = '{0}{1}.txt'.format(STOP_WORDS_DIR, language)
         with open(language_filename, 'rb') as language_file:
             stop_words = [line.decode('utf-8').strip()
                           for line in language_file.readlines()]
     except IOError:
         raise StopWordError(
-            '"%s" file is unreadable, check your installation.' %
-            language_filename)
+            '{0}" file is unreadable, check your installation.'.format(
+                language_filename
+            )
+        )
 
     STOP_WORDS_CACHE[language] = stop_words
 

stop_words/stop-words

@@ -1,15 +1,18 @@
 """
 Tests for stop-words
 """
+import random
 from unittest import TestCase
 from unittest import TestSuite
 from unittest import TestLoader
 
 import stop_words
-from stop_words import StopWordError
 from stop_words import get_stop_words
 from stop_words import safe_get_stop_words
+from stop_words import StopWordError
 from stop_words import STOP_WORDS_CACHE
+from stop_words import LANGUAGE_MAPPING
+from stop_words import AVAILABLE_LANGUAGES
 
 
 class StopWordsTestCase(TestCase):
@@ -29,7 +32,7 @@ def test_get_stop_words_cache(self):
         sw = get_stop_words('fr')
         self.assertTrue('french' in STOP_WORDS_CACHE)
         original_stop_words_dir = stop_words.STOP_WORDS_DIR
-        stop_words.STOP_WORDS_DIR = '/trash/'
+        stop_words.STOP_WORDS_DIR = 'not-existing-directory'
         self.assertEqual(sw, get_stop_words('french'))
         stop_words.STOP_WORDS_DIR = original_stop_words_dir
         try:
@@ -43,14 +46,23 @@ def test_get_stop_words_unavailable_language(self):
 
     def test_get_stop_words_install_issue(self):
         original_stop_words_dir = stop_words.STOP_WORDS_DIR
-        stop_words.STOP_WORDS_DIR = '/trash/'
+        stop_words.STOP_WORDS_DIR = 'not-existing-directory'
         self.assertRaises(StopWordError, get_stop_words, 'german')
         stop_words.STOP_WORDS_DIR = original_stop_words_dir
 
     def test_safe_get_stop_words(self):
         self.assertRaises(StopWordError, get_stop_words, 'huttese')
         self.assertEqual(safe_get_stop_words('huttese'), [])
 
+    def test_random_language_stop_words_load(self):
+        languages = list(LANGUAGE_MAPPING.keys()) + list(AVAILABLE_LANGUAGES)
+        sample = random.sample(languages, len(languages))
+        for language in sample:
+            stop_words = safe_get_stop_words(language)
+            self.assertTrue(
+                len(stop_words) > 0,
+                'Cannot load stopwords for {0} language'.format(language)
+            )
 
 loader = TestLoader()