Add support for ZWJ

Author: mpcabd

MANIFEST.in

@@ -1,3 +1,7 @@
-include arabic_reshaper/default-config.ini
+include arabic_reshaper/__init__.py
+include arabic_reshaper/__version__.py
 include arabic_reshaper/arabic_reshaper.py
-include README
+include arabic_reshaper/default-config.ini
+include arabic_reshaper/letters.py
+include arabic_reshaper/ligatures.py
+include README.md

arabic_reshaper/__init__.py

@@ -1 +1,5 @@
 from .arabic_reshaper import reshape, default_reshaper, ArabicReshaper
+
+import os
+
+exec(open(os.path.join(os.path.dirname(__file__), '__version__.py')).read())

arabic_reshaper/__version__.py

@@ -0,0 +1 @@
+__version__ = '2.0.9'

arabic_reshaper/arabic_reshaper.py

@@ -166,9 +166,13 @@ def reshape(self, text):
 
         delete_harakat = self.configuration.getboolean('delete_harakat')
         delete_tatweel = self.configuration.getboolean('delete_tatweel')
+        support_zwj = self.configuration.getboolean('support_zwj')
         positions_harakat = {}
 
-        for letter in text:
+        arabic_word_start = -1
+        zwjs = []
+
+        for i, letter in enumerate(text):
             if HARAKAT_RE.match(letter):
                 if not delete_harakat:
                     position = len(output) - 1
@@ -177,13 +181,51 @@ def reshape(self, text):
                     positions_harakat[position].append(letter)
             elif letter == TATWEEL and delete_tatweel:
                 pass
+            elif letter == ZWJ and support_zwj:
+                zwjs.append(i)
+
+                if arabic_word_start != -1:
+                    # Handle three consecutive ZWJs or more
+                    if (
+                        len(zwjs) > 2 and
+                        zwjs[-2] == i - 1 and
+                        zwjs[-3] == i - 2
+                    ):
+                        arabic_word_start = -1
+                    # Handle when previous letter is not ZWJ
+                    elif (
+                        output and
+                        len(zwjs) == 1 or (len(zwjs) > 1 and zwjs[-2] != i - 1)
+                    ):
+                        previous_letter = output[-1]
+                        if connects_with_letter_after(previous_letter[LETTER]):
+                            if previous_letter[FORM] == ISOLATED:
+                                output[-1] = (
+                                    previous_letter[LETTER],
+                                    INITIAL
+                                )
+                            else:
+                                output[-1] = (
+                                    previous_letter[LETTER],
+                                    MEDIAL
+                                )
             elif letter not in LETTERS:
+                arabic_word_start = -1
                 output.append((letter, NOT_SUPPORTED))
-            elif not output:
+            elif not output:  # first letter
+                arabic_word_start = i
                 output.append((letter, ISOLATED))
             else:
+                if arabic_word_start == -1:
+                    arabic_word_start = i
                 previous_letter = output[-1]
-                if previous_letter[FORM] == NOT_SUPPORTED:
+                if (
+                    arabic_word_start != i and
+                    zwjs and
+                    connects_with_letter_before(letter)
+                ):
+                    output.append((letter, FINAL))
+                elif previous_letter[FORM] == NOT_SUPPORTED:
                     output.append((letter, ISOLATED))
                 elif not connects_with_letter_before(letter):
                     output.append((letter, ISOLATED))
@@ -211,6 +253,10 @@ def reshape(self, text):
                     )
                     output.append((letter, FINAL))
 
+            # clear ZWJs
+            if zwjs and letter != ZWJ:
+                zwjs = []
+
         if self.configuration.getboolean('support_ligatures'):
             # Clean text from Harakat to be able to find ligatures
             text = HARAKAT_RE.sub('', text)

arabic_reshaper/default-config.ini

@@ -9,6 +9,9 @@ delete_harakat = yes
 # Whether to delete the Tatweel (U+0640) before reshaping or not.
 delete_tatweel = no
 
+# Whether to support ZWJ (U+200D) or not.
+support_zwj = yes
+
 # Whether to use ligatures or not.
 # Serves as a shortcut to disable all ligatures.
 support_ligatures = yes

arabic_reshaper/letters.py

@@ -21,6 +21,7 @@
 FINAL = 3
 
 TATWEEL = '\u0640'
+ZWJ = '\u200D'
 
 LETTERS = {
     # ARABIC LETTER HAMZA

arabic_reshaper/tests/test_002_reshaping.py

@@ -1,9 +1,12 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import unicode_literals
+from __future__ import print_function
 
 import unittest
+import sys
 import arabic_reshaper
+import arabic_reshaper.letters as letters
 
 
 def _reshaping_test(test):
@@ -13,6 +16,7 @@ def t(): test.assertEqual(case[1], test.reshaper.reshape(case[0]))
             with test.subTest(i=i, case=case[0]):
                 t()
         else:
+            print('running test case %d' % i, file=sys.stderr)
             t()
 
 
@@ -42,6 +46,70 @@ def test_reshaping(self):
         _reshaping_test(self)
 
 
+class TestZWJReshaping(unittest.TestCase):
+    def setUp(self):
+        self.reshaper = arabic_reshaper.default_reshaper
+
+        BEH = 'ب'
+        BEH_ISOLATED = letters.LETTERS[BEH][letters.ISOLATED]
+        BEH_INITIAL = letters.LETTERS[BEH][letters.INITIAL]
+        BEH_MEDIAL = letters.LETTERS[BEH][letters.MEDIAL]
+        BEH_FINAL = letters.LETTERS[BEH][letters.FINAL]
+
+        ALEF = 'ا'
+        ALEF_ISOLATED = letters.LETTERS[ALEF][letters.ISOLATED]
+        ALEF_FINAL = letters.LETTERS[ALEF][letters.FINAL]
+
+        HAMZA = 'ء'
+        HAMZA_ISOLATED = letters.LETTERS[HAMZA][letters.ISOLATED]
+
+        self.cases = (
+            (
+                BEH + HAMZA,
+                BEH_ISOLATED + HAMZA_ISOLATED
+            ),
+            (
+                letters.ZWJ + BEH + HAMZA,
+                BEH_ISOLATED + HAMZA_ISOLATED
+            ),
+            (
+                BEH + letters.ZWJ + HAMZA,
+                BEH_INITIAL + HAMZA_ISOLATED
+            ),
+            (
+                BEH + ALEF,
+                BEH_INITIAL + ALEF_FINAL
+            ),
+            (
+                BEH + letters.ZWJ + ALEF,
+                BEH_INITIAL + ALEF_FINAL
+            ),
+            (
+                BEH + letters.ZWJ + ALEF + letters.ZWJ,
+                BEH_INITIAL + ALEF_FINAL
+            ),
+            (
+                BEH + ALEF + BEH,
+                BEH_INITIAL + ALEF_FINAL + BEH_ISOLATED
+            ),
+            (
+                BEH + letters.ZWJ + ALEF + letters.ZWJ + BEH,
+                BEH_INITIAL + ALEF_FINAL + BEH_FINAL
+            ),
+            (
+                BEH + letters.ZWJ + HAMZA + BEH,
+                BEH_INITIAL + HAMZA_ISOLATED + BEH_ISOLATED
+            ),
+            (
+                BEH + letters.ZWJ + HAMZA + letters.ZWJ + BEH,
+                BEH_INITIAL + HAMZA_ISOLATED + BEH_FINAL
+            ),
+        )
+
+    def test_reshaping(self):
+        _reshaping_test(self)
+
+
 class TestReshapingWithHarakat(unittest.TestCase):
     def setUp(self):
         self.reshaper = arabic_reshaper.ArabicReshaper({

setup.py

@@ -3,11 +3,21 @@
 
 from setuptools import setup
 
+import os
+
+exec(
+    open(os.path.join(
+        os.path.dirname(__file__),
+        'arabic_reshaper',
+        '__version__.py'
+    )).read()
+)
+
 setup(
     name="arabic_reshaper",
     description=("Reconstruct Arabic sentences to be used in"
                  " applications that don't support Arabic"),
-    version='2.0.8',
+    version=__version__,
     platforms="ALL",
     license="GPL",
     packages=['arabic_reshaper'],