Merge branch 'rmeta-allow-gzip' of https://github.com/carantunes/tika-python into merge-pr316

Author: chrismattmann

.travis.yml

@@ -1,9 +1,15 @@
 language: python
+
 python:
   - "2.7"
   - "3.4"
   - "3.5"
   - "3.7"
-install: "pip install -r requirements.txt"
+
+install:
+  - "travis_retry pip install -r requirements.txt"
+  - "travis_retry pip install -e .[all]"
+
 script: py.test --cov=tika
+
 after_success: coveralls

README.md

@@ -92,6 +92,37 @@ parsed = parser.from_file('/path/to/file', 'http://tika:9998/tika')
 string_parsed = parser.from_buffer('Good evening, Dave', 'http://tika:9998/tika')
 ```
 
+You can also pass a binary stream
+```
+with open(file, 'rb') as file_obj:
+    response = tika.parser.from_file(file_obj)
+```
+
+Gzip compression
+---------------------
+Since Tika 1.24.1 gzip compression of input and output streams is allowed. 
+
+Input compression can be achieved with gzip or zlib:
+```
+    import zlib 
+
+    with open(file, 'rb') as file_obj:
+        return tika.parser.from_buffer(zlib.compress(file_obj.read()))
+
+...
+
+    import gzip
+ 
+    with open(file, 'rb') as file_obj:
+        return tika.parser.from_buffer(gzip.compress(file_obj.read()))
+```
+
+And output with the header:
+```
+    with open(file, 'rb') as file_obj:
+        return tika.parser.from_file(file_obj, headers={'Accept-Encoding': 'gzip, deflate'})
+```
+
 Specify Output Format To XHTML
 ---------------------
 The parser interface is optionally able to output the content as XHTML rather than plain text.

setup.py

@@ -19,11 +19,13 @@
 # $Id$
 
 import os.path
-import tika
 from io import open
 
+import tika
+
 try:
     from ez_setup import use_setuptools
+
     use_setuptools()
 except ImportError:
     pass
@@ -53,11 +55,25 @@
     'Topic :: Software Development :: Libraries :: Python Modules',
 ]
 
+
 def read(*rnames):
     return open(os.path.join(os.path.dirname(__file__), *rnames)).read()
 
+
 long_description = _descr
 
+extras_require = {
+    'tests': [
+        'memory-profiler>=0.57.0',
+        'pytest-benchmark>=3.2.2'
+    ],
+    'all': [
+    ]
+}
+
+for reqs in extras_require.values():
+    extras_require['all'].extend(reqs)
+
 setup(
     name='tika',
     version=version,
@@ -80,14 +96,13 @@ def read(*rnames):
             'tika-python = tika.tika:main'
         ],
     },
-    package_data = {
+    package_data={
         # And include any *.conf files found in the 'conf' subdirectory
         # for the package
     },
     install_requires=[
         'setuptools',
         'requests'
     ],
-    extras_require={
-    },
+    extras_require=extras_require,
 )

tika/__init__.py

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "1.24"
+__version__ = "1.24.1"
 
 try:
     __import__('pkg_resources').declare_namespace(__name__)

tika/tests/memory_benchmark.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+# encoding: utf-8
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# To run:
+# python tika/tests/memory_benchmark.py
+import os
+import zlib
+
+
+import tika.parser
+import tika.tika
+from memory_profiler import profile
+
+from tika.tests.utils import gzip_compress
+
+
+@profile
+def test_parser_binary():
+    """parse file binary"""
+    file = os.path.join(os.path.dirname(__file__), 'files', 'rwservlet.pdf')
+
+    with open(file, 'rb') as file_obj:
+        response = tika.parser.from_file(file_obj, headers={'Accept-Encoding': 'gzip, deflate'})
+
+
+@profile
+def test_parser_buffer():
+    """parse buffer"""
+    file = os.path.join(os.path.dirname(__file__), 'files', 'rwservlet.pdf')
+
+    with open(file, 'rb') as file_obj:
+        response = tika.parser.from_buffer(file_obj.read(), headers={'Accept-Encoding': 'gzip, deflate'})
+
+
+@profile
+def test_parser_zlib():
+    """parse buffer zlib"""
+
+    file = os.path.join(os.path.dirname(__file__), 'files', 'rwservlet.pdf')
+
+    with open(file, 'rb') as file_obj:
+        response = tika.parser.from_buffer(zlib.compress(file_obj.read()), headers={'Accept-Encoding': 'gzip, deflate'})
+
+
+@profile
+def test_parser_gzip():
+    """parse buffer gzip"""
+    file = os.path.join(os.path.dirname(__file__), 'files', 'rwservlet.pdf')
+
+    with open(file, 'rb') as file_obj:
+        response = tika.parser.from_buffer(gzip_compress(file_obj.read()), headers={'Accept-Encoding': 'gzip, deflate'})
+
+if __name__ == '__main__':
+    test_parser_buffer()
+    test_parser_binary()
+    test_parser_zlib()
+    test_parser_gzip()

tika/tests/test_benchmark.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# encoding: utf-8
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# pytest --benchmark-enable --benchmark-timer=time.process_time tika/tests/test_benchmark.py
+# pytest --benchmark-enable --benchmark-timer=time.process_time tika/tests/test_benchmark.py
+import os
+import unittest
+import zlib
+
+import tika.parser
+import tika.tika
+from tika.tests.utils import HTTPStatusOk, gzip_compress
+
+
+def test_local_binary(benchmark):
+    """parse file binary"""
+    file = os.path.join(os.path.dirname(__file__), 'files', 'rwservlet.pdf')
+    response = benchmark(tika_from_binary, file)
+
+    assert response['status'] == HTTPStatusOk
+
+
+def test_parser_buffer(benchmark):
+    """example how to send gzip file"""
+    file = os.path.join(os.path.dirname(__file__), 'files', 'rwservlet.pdf')
+    response = benchmark(tika_from_buffer, file)
+
+    assert response['status'] == HTTPStatusOk
+
+
+def test_parser_buffer_zlib_input(benchmark):
+    """example how to send gzip file"""
+    file = os.path.join(os.path.dirname(__file__), 'files', 'rwservlet.pdf')
+
+    response = benchmark(tika_from_buffer_zlib, file)
+
+    assert response['status'] == HTTPStatusOk
+
+
+def test_parser_buffer_gzip_input(benchmark):
+    """parse file binary"""
+    file = os.path.join(os.path.dirname(__file__), 'files', 'rwservlet.pdf')
+    response = benchmark(tika_from_buffer_gzip, file)
+
+    assert response['status'] == HTTPStatusOk
+
+
+def test_local_binary_with_gzip_output(benchmark):
+    """parse file binary"""
+    file = os.path.join(os.path.dirname(__file__), 'files', 'rwservlet.pdf')
+    response = benchmark(tika_from_binary, file, headers={'Accept-Encoding': 'gzip, deflate'})
+
+    assert response['status'] == HTTPStatusOk
+
+
+def test_parser_buffer_with_gzip_output(benchmark):
+    """example how to send gzip file"""
+    file = os.path.join(os.path.dirname(__file__), 'files', 'rwservlet.pdf')
+    response = benchmark(tika_from_buffer, file, headers={'Accept-Encoding': 'gzip, deflate'})
+
+    assert response['status'] == HTTPStatusOk
+
+
+def test_parser_buffer_zlib_input_and_gzip_output(benchmark):
+    """example how to send gzip file"""
+    file = os.path.join(os.path.dirname(__file__), 'files', 'rwservlet.pdf')
+
+    response = benchmark(tika_from_buffer_zlib, file, headers={'Accept-Encoding': 'gzip, deflate'})
+
+    assert response['status'] == HTTPStatusOk
+
+
+def test_parser_buffer_gzip_input_and_gzip_output(benchmark):
+    """parse file binary"""
+    file = os.path.join(os.path.dirname(__file__), 'files', 'rwservlet.pdf')
+    response = benchmark(tika_from_buffer_gzip, file, headers={'Accept-Encoding': 'gzip, deflate'})
+
+    assert response['status'] == HTTPStatusOk
+
+
+def tika_from_buffer_zlib(file, headers=None):
+    with open(file, 'rb') as file_obj:
+        return tika.parser.from_buffer(zlib.compress(file_obj.read()), headers=headers)
+
+
+def tika_from_buffer_gzip(file, headers=None):
+    with open(file, 'rb') as file_obj:
+        return tika.parser.from_buffer(gzip_compress(file_obj.read()), headers=headers)
+
+
+def tika_from_buffer(file, headers=None):
+    with open(file, 'rb') as file_obj:
+        return tika.parser.from_buffer(file_obj.read(), headers=headers)
+
+
+def tika_from_binary(file, headers=None):
+    with open(file, 'rb') as file_obj:
+        return tika.parser.from_file(file_obj, headers=headers)
+
+
+if __name__ == '__main__':
+    unittest.main()

tika/tests/test_tika.py

@@ -18,8 +18,10 @@
 # python -m unittest tests.tests
 import os
 import unittest
+
 import tika.parser
 import tika.tika
+from tika.tests.utils import HTTPStatusOk
 
 
 class CreateTest(unittest.TestCase):
@@ -50,6 +52,10 @@ def test_local_binary(self):
         with open(file, 'rb') as file_obj:
             self.assertTrue(tika.parser.from_file(file_obj))
 
+    def test_local_buffer(self):
+        response = tika.parser.from_buffer('Good evening, Dave')
+        self.assertEqual(response['status'], HTTPStatusOk)
+
     def test_local_path(self):
         """parse file path"""
         file = os.path.join(os.path.dirname(__file__), 'files', 'rwservlet.pdf')
@@ -62,5 +68,6 @@ def test_kill_server(self):
             tika.parser.from_file(file_obj)
         self.assertIsNone(tika.tika.killServer())
 
+
 if __name__ == '__main__':
     unittest.main()

tika/tests/utils.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+# encoding: utf-8
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import gzip
+
+
+def HTTPStatusOk():
+    try:
+        # python 2.7
+        import httplib
+
+        return httplib.OK
+    except ImportError:
+        try:
+            # python > 3.4
+            from http import HTTPStatus
+
+            return HTTPStatus.OK
+        except ImportError:
+            # python 3.4
+            import http.client
+
+            return http.client.OK
+
+
+HTTPStatusOk = HTTPStatusOk()
+
+
+def gzip_compress(file_obj):
+    try:
+        # python > 3.4
+        return gzip.compress(file_obj)
+    except AttributeError:
+        # python 2.7
+        import StringIO
+        out = StringIO.StringIO()
+        gzip_s = gzip.GzipFile(fileobj=out, mode="wb")
+        gzip_s.write(file_obj.encode('utf-8'))
+        gzip_s.close()
+
+        # Get the bytes written to the underlying file object
+        return out.getvalue()