Merge branch 'master' of github.com:chrismattmann/tika-python into kill-server

Author: igormp

README.md

@@ -5,9 +5,9 @@ tika-python
 ===========
 A Python port of the [Apache Tika](http://tika.apache.org/)
 library that makes Tika available using the
-[Tika REST Server](http://wiki.apache.org/tika/TikaJAXRS). 
+[Tika REST Server](http://wiki.apache.org/tika/TikaJAXRS).
 
-This makes Apache Tika available as a Python library, 
+This makes Apache Tika available as a Python library,
 installable via Setuptools, Pip and Easy Install.
 
 To use this library, you need to have Java 7+ installed on your
@@ -22,8 +22,14 @@ Installation (with pip)
 
 Installation (without pip)
 --------------------------
-1. `python setup.py build`  
-2. `python setup.py install`  
+1. `python setup.py build`
+2. `python setup.py install`
+
+Airgap Environment Setup
+------------------------
+To get this working in a disconnected environment, download a tika server file and set the TIKA_SERVER_JAR environment variable to TIKA_SERVER_JAR="file:///<yourpath>/tika-server.jar" which successfully tells `python-tika` to "download" this file and move it to `/tmp/tika-server.jar` and run as background process.
+
+This is the only way to run `python-tika` without internet access. Without this set, the default is to check the tika version and pull latest every time from Apache.
 
 Environment Variables
 ---------------------
@@ -58,11 +64,11 @@ print(parsed["content"])
 
 Parser Interface
 ----------------------
-The parser interface extracts text and metadata using the /rmeta 
+The parser interface extracts text and metadata using the /rmeta
 interface. This is one of the better ways to get the internal XHTML
 content extracted.
 
-Note: 
+Note:
 ![Alert Icon](https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon28.png "Alert")
 The parser interface needs the following environment variable set on the console for printing of the extracted content.
 ```export PYTHONIOENCODING=utf8```
@@ -85,7 +91,7 @@ Specify Output Format To XHTML
 ---------------------
 The parser interface is optionally able to output the content as XHTML rather than plain text.
 
-Note: 
+Note:
 ![Alert Icon](https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon28.png "Alert")
 The parser interface needs the following environment variable set on the console for printing of the extracted content.
 ```export PYTHONIOENCODING=utf8```
@@ -129,7 +135,7 @@ print(detector.from_file('/path/to/file'))
 Config Interface
 ----------------------
 The config interface allows you to inspect the Tika Server environment's
-configuration including what parsers, mime types, and detectors the 
+configuration including what parsers, mime types, and detectors the
 server has been configured with.
 
 ```
@@ -143,7 +149,7 @@ print(config.getDetectors())
 
 Language Detection Interface
 ---------------------------------
-The language detection interface provides a 2 character language 
+The language detection interface provides a 2 character language
 code texted based on the text in provided file.
 
 ```
@@ -187,10 +193,10 @@ Changing the Tika Classpath
 ---------------------------
 You can update the classpath that Tika server uses by
 setting the classpath as a set of ':' delimited strings.
-For example if you want to get Tika-Python working with 
+For example if you want to get Tika-Python working with
 [GeoTopicParsing](http://wiki.apache.org/tika/GeoTopicParser),
 you can do this, replace paths below with your own paths, as
-identified [here](http://wiki.apache.org/tika/GeoTopicParser) 
+identified [here](http://wiki.apache.org/tika/GeoTopicParser)
 and make sure that you have done this:
 
 kill Tika server (if already running):
@@ -294,6 +300,7 @@ Contributors
 * Igor Tokarev, Freelance
 * Imraan Parker, Freelance
 * Annie K. Didier, JPL
+* Juan Elosua, TEGRA Cybersecurity Center
 
 Thanks
 ======

setup.py

@@ -20,6 +20,7 @@
 
 import os.path
 import tika
+from io import open
 
 try:
     from ez_setup import use_setuptools

tika/__init__.py

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "1.22"
+__version__ = "1.23"
 
 try:
     __import__('pkg_resources').declare_namespace(__name__)

tika/parser.py

@@ -20,10 +20,10 @@
 import os
 import json
 
-def from_file(filename, serverEndpoint=ServerEndpoint, xmlContent=False, headers=None, config_path=None, requestOptions={}):
+def from_file(filename, service='all', serverEndpoint=ServerEndpoint, xmlContent=False, headers=None, config_path=None, requestOptions={}):
     '''
     Parses a file for metadata and content
-    :param filename: path to file which needs to be parsed
+    :param filename: path to file which needs to be parsed or binary file using open(path,'rb')
     :param serverEndpoint: Server endpoint url
     :param xmlContent: Whether or not XML content be requested.
                     Default is 'False', which results in text content.
@@ -33,11 +33,11 @@ def from_file(filename, serverEndpoint=ServerEndpoint, xmlContent=False, headers
             'content' has a str value and metadata has a dict type value.
     '''
     if not xmlContent:
-        jsonOutput = parse1('all', filename, serverEndpoint, headers=headers, config_path=config_path, requestOptions=requestOptions)
+        output = parse1(service, filename, serverEndpoint, headers=headers, config_path=config_path, requestOptions=requestOptions)
     else:
-        jsonOutput = parse1('all', filename, serverEndpoint, services={'meta': '/meta', 'text': '/tika', 'all': '/rmeta/xml'},
+        output = parse1(service, filename, serverEndpoint, services={'meta': '/meta', 'text': '/tika', 'all': '/rmeta/xml'},
                             headers=headers, config_path=config_path, requestOptions=requestOptions)
-    return _parse(jsonOutput)
+    return _parse(output, service)
 
 
 def from_buffer(string, serverEndpoint=ServerEndpoint, xmlContent=False, headers=None, config_path=None, requestOptions={}):
@@ -61,20 +61,35 @@ def from_buffer(string, serverEndpoint=ServerEndpoint, xmlContent=False, headers
 
     return _parse((status,response))
 
-def _parse(jsonOutput):
+def _parse(output, service='all'):
     '''
-    Parses JSON response from Tika REST API server
-    :param jsonOutput: JSON output from Tika Server
+    Parses response from Tika REST API server
+    :param output: output from Tika Server
+    :param service: service requested from the tika server
+                    Default is 'all', which results in recursive text content+metadata.
+                    'meta' returns only metadata
+                    'text' returns only content
     :return: a dictionary having 'metadata' and 'content' values
     '''
-    parsed={}
-    if not jsonOutput:
+    parsed={'metadata': None, 'content': None}
+    if not output:
         return parsed
-    
-    parsed["status"] = jsonOutput[0]
-    if jsonOutput[1] == None or jsonOutput[1] == "":
+
+    parsed["status"] = output[0]
+    if output[1] == None or output[1] == "":
+        return parsed
+
+    if service == "text":
+        parsed["content"] = output[1]
+        return parsed
+
+    realJson = json.loads(output[1])
+
+    parsed["metadata"] = {}
+    if service == "meta":
+        for key in realJson:
+            parsed["metadata"][key] = realJson[key]
         return parsed
-    realJson = json.loads(jsonOutput[1])
 
     content = ""
     for js in realJson:
@@ -85,7 +100,6 @@ def _parse(jsonOutput):
         content = None
 
     parsed["content"] = content
-    parsed["metadata"] = {}
 
     for js in realJson:
         for n in js:

tika/tests/files/rwservlet.pdf

@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+# encoding: utf-8
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# python -m unittest tika.tests.test_from_file_service
+
+import unittest
+import tika.parser
+
+
+class CreateTest(unittest.TestCase):
+    'test different services in from_file parsing: Content, Metadata or both in recursive mode'
+
+    def test_default_service(self):
+        'parse file using default service'
+        result = tika.parser.from_file(
+            'https://boe.es/boe/dias/2019/12/02/pdfs/BOE-A-2019-17288.pdf')
+        self.assertEqual(result['metadata']['Content-Type'],'application/pdf')
+        self.assertIn('AUTORIDADES Y PERSONAL',result['content'])
+    def test_default_service_explicit(self):
+        'parse file using default service explicitly'
+        result = tika.parser.from_file(
+            'https://boe.es/boe/dias/2019/12/02/pdfs/BOE-A-2019-17288.pdf', service='all')
+        self.assertEqual(result['metadata']['Content-Type'],'application/pdf')
+        self.assertIn('AUTORIDADES Y PERSONAL',result['content'])
+    def test_text_service(self):
+        'parse file using the content only service'
+        result = tika.parser.from_file(
+            'https://boe.es/boe/dias/2019/12/02/pdfs/BOE-A-2019-17288.pdf', service='text')
+        self.assertIsNone(result['metadata'])
+        self.assertIn('AUTORIDADES Y PERSONAL',result['content'])
+    def test_meta_service(self):
+        'parse file using the content only service'
+        result = tika.parser.from_file(
+            'https://boe.es/boe/dias/2019/12/02/pdfs/BOE-A-2019-17288.pdf', service='meta')
+        self.assertIsNone(result['content'])
+        self.assertEqual(result['metadata']['Content-Type'],'application/pdf')
+    def test_invalid_service(self):
+        'parse file using an invalid service should perform the default parsing'
+        result = tika.parser.from_file(
+            'https://boe.es/boe/dias/2019/12/02/pdfs/BOE-A-2019-17288.pdf', service='bad')
+        self.assertEqual(result['metadata']['Content-Type'],'application/pdf')
+        self.assertIn('AUTORIDADES Y PERSONAL',result['content'])
+
+if __name__ == '__main__':
+    unittest.main()

tika/tests/test_from_file_service.py

@@ -15,32 +15,45 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # 
-#python -m unittest tests.tests
-
+# python -m unittest tests.tests
+import os
 import unittest
 import tika.parser
 
 
 class CreateTest(unittest.TestCase):
-    "test for file types"
+    """test for file types"""
 
     def test_remote_pdf(self):
-        'parse remote PDF'
+        """parse remote PDF"""
         self.assertTrue(tika.parser.from_file(
             'http://appsrv.achd.net/reports/rwservlet?food_rep_insp&P_ENCOUNTER=201504160015'))
+
     def test_remote_html(self):
-        'parse remote HTML' 
-        self.assertTrue(tika.parser.from_file(
-            'http://neverssl.com/index.html'))
+        """parse remote HTML"""
+        self.assertTrue(tika.parser.from_file('http://neverssl.com/index.html'))
+
     def test_remote_mp3(self):
-        'parese remote mp3'
+        """parse remote mp3"""
         self.assertTrue(tika.parser.from_file(
             'https://archive.org/download/Ainst-Spaceshipdemo.mp3/Ainst-Spaceshipdemo.mp3'))
+
     def test_remote_jpg(self):
-        'parse remote jpg'
+        """parse remote jpg"""
         self.assertTrue(tika.parser.from_file(
             'https://www.nasa.gov/sites/default/files/thumbnails/image/j2m-shareable.jpg'))
 
+    def test_local_binary(self):
+        """parse file binary"""
+        file = os.path.join(os.path.dirname(__file__), 'files', 'rwservlet.pdf')
+        with open(file, 'rb') as file_obj:
+            self.assertTrue(tika.parser.from_file(file_obj))
+
+    def test_local_path(self):
+        """parse file path"""
+        file = os.path.join(os.path.dirname(__file__), 'files', 'rwservlet.pdf')
+        self.assertTrue(tika.parser.from_file(file))
+
 
 if __name__ == '__main__':
     unittest.main()

tika/tests/test_tika.py

@@ -58,6 +58,7 @@
     detected = detector.from_buffer('some buffered content', config_path='/path/to/configfile')
 
 '''
+import types
 
 USAGE = """
 tika.py [-v] [-e] [-o <outputDir>] [--server <TikaServerEndpoint>] [--install <UrlToTikaServerJar>] [--port <portNumber>] <command> <option> <urlOrPathToFile>
@@ -141,6 +142,7 @@ def make_content_disposition_header(fn):
 from os import walk
 import signal
 import logging
+import io
 
 log_path = os.getenv('TIKA_LOG_PATH', tempfile.gettempdir())
 log_file = os.path.join(log_path, 'tika.log')
@@ -162,7 +164,7 @@ def make_content_disposition_header(fn):
 log.setLevel(logging.INFO)
 
 Windows = True if platform.system() == "Windows" else False
-TikaVersion = os.getenv('TIKA_VERSION', '1.22')
+TikaVersion = os.getenv('TIKA_VERSION', '1.23')
 TikaJarPath = os.getenv('TIKA_PATH', tempfile.gettempdir())
 TikaFilesPath = tempfile.gettempdir()
 TikaServerLogFilePath = log_path
@@ -328,9 +330,11 @@ def parse1(option, urlOrPath, serverEndpoint=ServerEndpoint, verbose=Verbose, ti
         log.warning('config option must be one of meta, text, or all; using all.')
     service = services.get(option, services['all'])
     if service == '/tika': responseMimeType = 'text/plain'
-    headers.update({'Accept': responseMimeType, 'Content-Disposition': make_content_disposition_header(path)})
-    status, response = callServer('put', serverEndpoint, service, open(path, 'rb'),
-                                  headers, verbose, tikaServerJar, config_path=config_path, rawResponse=rawResponse, requestOptions=requestOptions)
+    headers.update({'Accept': responseMimeType, 'Content-Disposition': make_content_disposition_header(path.encode('utf-8') if type(path) is unicode_string else path)})
+    with urlOrPath if _is_file_object(urlOrPath) else open(path, 'rb') as f:
+        status, response = callServer('put', serverEndpoint, service, f,
+                                      headers, verbose, tikaServerJar, config_path=config_path,
+                                      rawResponse=rawResponse, requestOptions=requestOptions)
 
     if file_type == 'remote': os.unlink(path)
     return (status, response)
@@ -547,7 +551,6 @@ def callServer(verb, serverEndpoint, service, data, headers, verbose=Verbose, ti
     effectiveRequestOptions.update(requestOptions)
 
     resp = verbFn(serviceUrl, encodedData, **effectiveRequestOptions)
-    encodedData.close() # closes the file reading data
 
     if verbose:
         print(sys.stderr, "Request headers: ", headers)
@@ -701,14 +704,26 @@ def toFilename(url):
     value = re.sub(r'[^\w\s\.\-]', '-', path).strip().lower()
     return re.sub(r'[-\s]+', '-', value).strip("-")[-200:]
 
-    
+
+def _is_file_object(f):
+    try:
+        file_types = (types.FileType, io.IOBase)
+    except AttributeError:
+        file_types = (io.IOBase,)
+
+    return isinstance(f, file_types)
+
 def getRemoteFile(urlOrPath, destPath):
     '''
     Fetches URL to local path or just returns absolute path.
     :param urlOrPath: resource locator, generally URL or path
     :param destPath: path to store the resource, usually a path on file system
-    :return: tuple having (path, 'local'/'remote')
+    :return: tuple having (path, 'local'/'remote'/'binary')
     '''
+    # handle binary stream input
+    if _is_file_object(urlOrPath):
+        return (urlOrPath.name, 'binary')
+
     urlp = urlparse(urlOrPath)
     if urlp.scheme == '':
         return (os.path.abspath(urlOrPath), 'local')
@@ -774,8 +789,6 @@ def checkPortIsOpen(remoteServerHost=ServerHost, port = Port):
             return True
         else :
             return False
-        sock.close()
-        #FIXME: the above line is unreachable
 
     except KeyboardInterrupt:
         print("You pressed Ctrl+C")
@@ -789,6 +802,9 @@ def checkPortIsOpen(remoteServerHost=ServerHost, port = Port):
         print("Couldn't connect to server")
         sys.exit()
 
+    finally:
+        sock.close()
+
 def main(argv=None):
     """Run Tika from command line according to USAGE."""
     global Verbose