Merge pull request #171 from yarongon/hotfix/issue-169-uft8-problem

Fix the utf-8 issue on the unpack interface

Author: chrismattmann

tika/tests/tests_unpack.py

@@ -0,0 +1,31 @@
+# coding=utf8
+
+import unittest
+from tempfile import NamedTemporaryFile
+from tika import unpack
+
+
+class CreateTest(unittest.TestCase):
+    "Test different encodings"
+    text_utf8 = u"Hello, world!! 😎 👽"
+    text_ascii = u"Hello, world!!"
+
+    def test_utf8(self):
+        with NamedTemporaryFile("w+b", prefix='tika-python', suffix='.txt', dir='/tmp') as f:
+            f.write(self.text_utf8.encode("utf8"))
+            f.flush()
+            f.seek(0)
+            parsed = unpack.from_file(f.name)
+            self.assertEqual(parsed["content"].strip(), self.text_utf8)
+
+    def test_ascii(self):
+        with NamedTemporaryFile("w+t", prefix='tika-python', suffix='.txt', dir='/tmp') as f:
+            f.write(self.text_ascii)
+            f.flush()
+            f.seek(0)
+            parsed = unpack.from_file(f.name)
+            self.assertEqual(parsed["content"].strip(), self.text_ascii)
+
+
+if __name__ == '__main__':
+    unittest.main()

tika/unpack.py

@@ -28,6 +28,7 @@
 # tarfile returned object can be used as is in earlier versions.
 _text_wrapper = TextIOWrapper if version_info.major >= 3 else lambda x: x
 
+
 def from_file(filename, serverEndpoint=ServerEndpoint):
     '''
     Parse from file
@@ -36,7 +37,7 @@ def from_file(filename, serverEndpoint=ServerEndpoint):
     :return:
     '''
     tarOutput = parse1('unpack', filename, serverEndpoint,
-                       responseMimeType='application/x-tar', 
+                       responseMimeType='application/x-tar',
                        services={'meta': '/meta', 'text': '/tika',
                                  'all': '/rmeta/xml', 'unpack': '/unpack/all'},
                        rawResponse=True)
@@ -52,8 +53,8 @@ def from_buffer(string, serverEndpoint=ServerEndpoint):
     '''
     status, response = callServer('put', serverEndpoint, '/unpack/all', string,
                                   {'Accept': 'application/x-tar'}, False,
-                                  rawResponse=True)      
-  
+                                  rawResponse=True)
+
     return _parse((status, response))
 
 
@@ -96,7 +97,10 @@ def _parse(tarOutput):
 
         contentMember = tarFile.getmember("__TEXT__")
         if not contentMember.issym() and contentMember.isfile():
-            content = _text_wrapper(tarFile.extractfile(contentMember)).read()
+            if version_info.major >= 3:
+                content = _text_wrapper(tarFile.extractfile(contentMember), encoding='utf8').read()
+            else:
+                content = tarFile.extractfile(contentMember).read().decode('utf8')
 
     # get the remaining files as attachments
     attachments = {}