Merge pull request #269 from carantunes/parse-from-file-binary

Allow parsing from binary streams

Author: chrismattmann

tika/parser.py

@@ -23,7 +23,7 @@
 def from_file(filename, service='all', serverEndpoint=ServerEndpoint, xmlContent=False, headers=None, config_path=None, requestOptions={}):
     '''
     Parses a file for metadata and content
-    :param filename: path to file which needs to be parsed
+    :param filename: path to file which needs to be parsed or binary file using open(path,'rb')
     :param serverEndpoint: Server endpoint url
     :param xmlContent: Whether or not XML content be requested.
                     Default is 'False', which results in text content.

tika/tests/files/rwservlet.pdf

@@ -15,32 +15,45 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # 
-#python -m unittest tests.tests
-
+# python -m unittest tests.tests
+import os
 import unittest
 import tika.parser
 
 
 class CreateTest(unittest.TestCase):
-    "test for file types"
+    """test for file types"""
 
     def test_remote_pdf(self):
-        'parse remote PDF'
+        """parse remote PDF"""
         self.assertTrue(tika.parser.from_file(
             'http://appsrv.achd.net/reports/rwservlet?food_rep_insp&P_ENCOUNTER=201504160015'))
+
     def test_remote_html(self):
-        'parse remote HTML' 
-        self.assertTrue(tika.parser.from_file(
-            'http://neverssl.com/index.html'))
+        """parse remote HTML"""
+        self.assertTrue(tika.parser.from_file('http://neverssl.com/index.html'))
+
     def test_remote_mp3(self):
-        'parese remote mp3'
+        """parse remote mp3"""
         self.assertTrue(tika.parser.from_file(
             'https://archive.org/download/Ainst-Spaceshipdemo.mp3/Ainst-Spaceshipdemo.mp3'))
+
     def test_remote_jpg(self):
-        'parse remote jpg'
+        """parse remote jpg"""
         self.assertTrue(tika.parser.from_file(
             'https://www.nasa.gov/sites/default/files/thumbnails/image/j2m-shareable.jpg'))
 
+    def test_local_binary(self):
+        """parse file binary"""
+        file = os.path.join(os.path.dirname(__file__), 'files', 'rwservlet.pdf')
+        with open(file, 'rb') as file_obj:
+            self.assertTrue(tika.parser.from_file(file_obj))
+
+    def test_local_path(self):
+        """parse file path"""
+        file = os.path.join(os.path.dirname(__file__), 'files', 'rwservlet.pdf')
+        self.assertTrue(tika.parser.from_file(file))
+
 
 if __name__ == '__main__':
     unittest.main()

tika/tests/test_tika.py

@@ -58,6 +58,7 @@
     detected = detector.from_buffer('some buffered content', config_path='/path/to/configfile')
 
 '''
+import types
 
 USAGE = """
 tika.py [-v] [-e] [-o <outputDir>] [--server <TikaServerEndpoint>] [--install <UrlToTikaServerJar>] [--port <portNumber>] <command> <option> <urlOrPathToFile>
@@ -140,6 +141,7 @@ def make_content_disposition_header(fn):
 from subprocess import STDOUT
 from os import walk
 import logging
+import io
 
 log_path = os.getenv('TIKA_LOG_PATH', tempfile.gettempdir())
 log_file = os.path.join(log_path, 'tika.log')
@@ -325,9 +327,9 @@ def parse1(option, urlOrPath, serverEndpoint=ServerEndpoint, verbose=Verbose, ti
     service = services.get(option, services['all'])
     if service == '/tika': responseMimeType = 'text/plain'
     headers.update({'Accept': responseMimeType, 'Content-Disposition': make_content_disposition_header(path.encode('utf-8') if type(path) is unicode_string else path)})
-    with open(path, 'rb') as f:
+    with urlOrPath if _is_file_object(urlOrPath) else open(path, 'rb') as f:
         status, response = callServer('put', serverEndpoint, service, f,
-                                      headers, verbose, tikaServerJar, config_path=config_path, 
+                                      headers, verbose, tikaServerJar, config_path=config_path,
                                       rawResponse=rawResponse, requestOptions=requestOptions)
 
     if file_type == 'remote': os.unlink(path)
@@ -690,14 +692,26 @@ def toFilename(url):
     value = re.sub(r'[^\w\s\.\-]', '-', path).strip().lower()
     return re.sub(r'[-\s]+', '-', value).strip("-")[-200:]
 
-    
+
+def _is_file_object(f):
+    try:
+        file_types = (types.FileType, io.IOBase)
+    except AttributeError:
+        file_types = (io.IOBase,)
+
+    return isinstance(f, file_types)
+
 def getRemoteFile(urlOrPath, destPath):
     '''
     Fetches URL to local path or just returns absolute path.
     :param urlOrPath: resource locator, generally URL or path
     :param destPath: path to store the resource, usually a path on file system
-    :return: tuple having (path, 'local'/'remote')
+    :return: tuple having (path, 'local'/'remote'/'binary')
     '''
+    # handle binary stream input
+    if _is_file_object(urlOrPath):
+        return (urlOrPath.name, 'binary')
+
     urlp = urlparse(urlOrPath)
     if urlp.scheme == '':
         return (os.path.abspath(urlOrPath), 'local')