Merge master and resolve conflicts

Author: thammegowda

README.md

@@ -52,8 +52,12 @@ print(parsed["metadata"])
 print(parsed["content"])
 ```
 
-Parser Interface (new)
+Parser Interface
 ----------------------
+The parser interface extracts text and metadata using the /rmeta 
+interface. THis is one of the better ways to get the internal XHTML
+content extracted.
+
 ```
 #!/usr/bin/env python
 import tika
@@ -68,17 +72,37 @@ parsed = parser.from_file('/path/to/file', 'http://tika:9998/tika')
 string_parsed = parser.from_buffer('Good evening, Dave', 'http://tika:9998/tika')
 ```
 
-Detect Interface (new)
+Unpack Interface
+----------------
+The unpack interface handles both metadata and text extraction in a single
+call and internally returns back a tarball of metadata and text entries that
+is internally unpacked, reducing the wire load for extraction.
+
+```
+#!/usr/bin/env python
+import tika
+from tika import unpack
+parsed = unpack.from_file('/path/to/file')
+```
+
+Detect Interface
 ----------------------
+The detect interface provides a IANA MIME type classification for the
+provided file.
+
 ```
 #!/usr/bin/env python
 import tika
 from tika import detector
 print(detector.from_file('/path/to/file'))
 ```
 
-Config Interface (new)
+Config Interface
 ----------------------
+The config interface allows you to inspect the Tika Server environment's
+configuration including what parsers, mime types, and detectors the 
+server has been configured with.
+
 ```
 #!/usr/bin/env python
 import tika
@@ -88,16 +112,22 @@ print(config.getMimeTypes())
 print(config.getDetectors())
 ```
 
-Language Detection Interface (new)
+Language Detection Interface
 ---------------------------------
+The language detection interface provides a 2 character language 
+code texted based on the text in provided file.
+
 ```
 #!/usr/bin/env python
 from tika import language
 print(language.from_file('/path/to/file'))
 ```
 
-Translate Interface (new)
+Translate Interface
 ------------------------
+The translate interface translates the text automatically extracted
+by Tika from the source language to the destination language.
+
 ```
 #!/usr/bin/env python
 from tika import translate

tika/__init__.py

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "1.14"
+__version__ = "1.14.2"
 
 try:
     __import__('pkg_resources').declare_namespace(__name__)

tika/tests/arguments/test_remote_content.csv

@@ -3,7 +3,6 @@
 ,"https://catalog.data.gov/dataset/free-application-for-federal-student-aid-201213/resource/f2437ade-d39f-40fb-9dba-19c3b26a8662",,
 ,"https://studentaid.ed.gov/sites/default/files/fsawg/datacenter/library/FAFSAReportDefinitions.doc",,"FAFSAReportDefinitions.doc"
 ,"http://open.defense.gov/Portals/23/Documents/FOIA_WebsiteCompliance.ppt",,"FOIA_WebsiteCompliance.ppt"
-,"http://www.ars.usda.gov/is/video/asx/freeze.asx",,"freeze.asx"
 ,"https://catalog.data.gov/dataset/geologic-map-of-the-state-of-hawaii",,"geologic-map-of-the-state-of-hawaii"
 ,"http://data.octo.dc.gov/Attachment.aspx?where=Citywide&area=&what=CSV&date=Issueddate&from=4/12/2015%2012:00:00%20AM&to=4/19/2015%2010:00:00%20PM&dataset=DCRA_PERMIT&datasetid=5&whereInd=0&areaInd=0&whatInd=0&dateInd=0&whenInd=0",,"DCRA_PERMIT__from04_12_2015__to04_19_2015.csv"
 ,"http://www.dmdc.osd.mil/Rank_Gender_Race.xls",,"Rank_Gender_Race.xls"

tika/tests/arguments/test_remote_metadata.csv

@@ -12,7 +12,6 @@
 ,"http://downloads.bbc.co.uk/podcasts/worldservice/6min_vocab/6min_vocab_20150511-1134a.mp3",,"6min_vocab_20150511-1134a.mp3"
 ,"http://www.ars.usda.gov/is/podcasts/tomatoes.mp4",,"tomatoes.mp4"
 ,"http://media.ars.usda.gov/is/mp4/freezeplants.mp4",,"freezeplants.mp4"
-,"http://www.ars.usda.gov/is/video/asx/freeze.asx",,"freeze.asx"
 ,"https://catalog.data.gov/dataset/geologic-map-of-the-state-of-hawaii",,"geologic-map-of-the-state-of-hawaii"
 ,"http://pubs.usgs.gov/of/2007/1089/Haw_St_tabfiles.zip",,"Haw_St_tabfiles.zip"
 ,"http://data.octo.dc.gov/Attachment.aspx?where=Citywide&area=&what=CSV&date=Issueddate&from=4/12/2015%2012:00:00%20AM&to=4/19/2015%2010:00:00%20PM&dataset=DCRA_PERMIT&datasetid=5&whereInd=0&areaInd=0&whatInd=0&dateInd=0&whenInd=0",,"DCRA_PERMIT__from04_12_2015__to04_19_2015.csv"

tika/tests/arguments/test_unsupported.csv

@@ -5,3 +5,4 @@ Content-Type,URL,Language,local
 ,http://www.defense.gov/multimedia/web_graphics/coastgrd/USCGb1.eps,,USCGb1.eps
 ,http://www.defense.gov/multimedia/web_graphics/coastgrd/USCGb.wmf,,USCGb.wmf
 ,http://www.ars.usda.gov/is/podcasts/tomatoes.wmv,,tomatoes.wmv
+,http://media.ars.usda.gov/is/wm/freeze.wmv,,freeze.wmv

tika/tika.py

@@ -136,7 +136,7 @@ def make_content_disposition_header(fn):
 log_file = os.path.join(log_path, 'tika.log')
 
 logFormatter = logging.Formatter("%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s]  %(message)s")
-log = logging.getLogger()
+log = logging.getLogger('tika.tika')
 
 # File logs
 fileHandler = logging.FileHandler(log_file)
@@ -272,7 +272,7 @@ def parseAndSave(option, urlOrPaths, outDir=None, serverEndpoint=ServerEndpoint,
 
 def parse(option, urlOrPaths, serverEndpoint=ServerEndpoint, verbose=Verbose, tikaServerJar=TikaServerJar, 
           responseMimeType='application/json',
-          services={'meta': '/meta', 'text': '/tika', 'all': '/rmeta'}):
+          services={'meta': '/meta', 'text': '/tika', 'all': '/rmeta'}, rawResponse=False):
     '''
     Parse the objects and return extracted metadata and/or text in JSON format.
     :param option:
@@ -289,7 +289,7 @@ def parse(option, urlOrPaths, serverEndpoint=ServerEndpoint, verbose=Verbose, ti
 
 def parse1(option, urlOrPath, serverEndpoint=ServerEndpoint, verbose=Verbose, tikaServerJar=TikaServerJar, 
           responseMimeType='application/json',
-          services={'meta': '/meta', 'text': '/tika', 'all': '/rmeta/text'}):
+          services={'meta': '/meta', 'text': '/tika', 'all': '/rmeta/text'}, rawResponse=False):
     '''
     Parse the object and return extracted metadata and/or text in JSON format.
     :param option:
@@ -307,11 +307,9 @@ def parse1(option, urlOrPath, serverEndpoint=ServerEndpoint, verbose=Verbose, ti
     service = services.get(option, services['all'])
     if service == '/tika': responseMimeType = 'text/plain'
     status, response = callServer('put', serverEndpoint, service, open(path, 'rb'),
-                                  {
-                                      'Accept': responseMimeType,
-                                      'Content-Disposition': make_content_disposition_header(path)
-                                  }, 
-                                  verbose, tikaServerJar)
+                                  {'Accept': responseMimeType, 'Content-Disposition': make_content_disposition_header(path)},
+                                  verbose, tikaServerJar, rawResponse=rawResponse)
+
 
     if file_type == 'remote': os.unlink(path)
     return (status, response)
@@ -479,8 +477,9 @@ def getConfig(option, serverEndpoint=ServerEndpoint, verbose=Verbose, tikaServer
     return (status, response)
 
 
-def callServer(verb, serverEndpoint, service, data, headers, verbose=Verbose, tikaServerJar=TikaServerJar, 
-               httpVerbs={'get': requests.get, 'put': requests.put, 'post': requests.post},classpath=None):
+def callServer(verb, serverEndpoint, service, data, headers, verbose=Verbose, tikaServerJar=TikaServerJar,
+               httpVerbs={'get': requests.get, 'put': requests.put, 'post': requests.post}, classpath=None,
+               rawResponse=False):
     '''
     Call the Tika Server, do some error checking, and return the response.
     :param verb:
@@ -522,8 +521,12 @@ def callServer(verb, serverEndpoint, service, data, headers, verbose=Verbose, ti
         print(sys.stderr, "Response headers: ", resp.headers)
     if resp.status_code != 200:
         log.warning('Tika server returned status: %d', resp.status_code)
+
     resp.encoding = "utf-8"
-    return (resp.status_code, resp.text)
+    if rawResponse:
+        return (resp.status_code, resp.content)
+    else:
+        return (resp.status_code, resp.text)
 
 
 def checkTikaServer(serverHost=ServerHost, port = Port, tikaServerJar=TikaServerJar,classpath=None):

tika/unpack.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+# encoding: utf-8
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from .tika import parse1, callServer, ServerEndpoint
+import tarfile
+from io import BytesIO, TextIOWrapper
+import csv
+from sys import version_info
+
+# Python 3 introduced .readable() to tarfile extracted files objects - this
+# is required to wrap a TextIOWrapper around the object. However, wrapping
+# with TextIOWrapper is only required for csv.reader() in Python 3, so the
+# tarfile returned object can be used as is in earlier versions.
+_text_wrapper = TextIOWrapper if version_info.major >= 3 else lambda x: x
+
+def from_file(filename, serverEndpoint=ServerEndpoint):
+    '''
+    Parse from file
+    :param filename: file
+    :param serverEndpoint: Tika server end point (optional)
+    :return:
+    '''
+    tarOutput = parse1('unpack', filename, serverEndpoint,
+                       responseMimeType='application/x-tar', 
+                       services={'meta': '/meta', 'text': '/tika',
+                                 'all': '/rmeta/xml', 'unpack': '/unpack/all'},
+                       rawResponse=True)
+    return _parse(tarOutput)
+
+
+def from_buffer(string, serverEndpoint=ServerEndpoint):
+    '''
+    Parse from buffered content
+    :param string:  buffered content
+    :param serverEndpoint: Tika server URL (Optional)
+    :return: parsed content
+    '''
+    status, response = callServer('put', serverEndpoint, '/unpack/all', string,
+                                  {'Accept': 'application/x-tar'}, False,
+                                  rawResponse=True)      
+  
+    return _parse((status, response))
+
+
+def _parse(tarOutput):
+    parsed = {}
+    if not tarOutput:
+        return parsed
+    elif tarOutput[1] is None or tarOutput[1] == b"":
+        return parsed
+
+    tarFile = tarfile.open(fileobj=BytesIO(tarOutput[1]))
+
+    # get the member names
+    memberNames = list(tarFile.getnames())
+
+    # extract the metadata
+    metadata = {}
+    if "__METADATA__" in memberNames:
+        memberNames.remove("__METADATA__")
+
+        metadataMember = tarFile.getmember("__METADATA__")
+        if not metadataMember.issym() and metadataMember.isfile():
+            metadataFile = _text_wrapper(tarFile.extractfile(metadataMember))
+            metadataReader = csv.reader(metadataFile)
+            for metadataLine in metadataReader:
+                # each metadata line comes as a key-value pair, with list values
+                # returned as extra values in the line - convert single values
+                # to non-list values to be consistent with parser metadata
+                assert len(metadataLine) >= 2
+
+                if len(metadataLine) > 2:
+                    metadata[metadataLine[0]] = metadataLine[1:]
+                else:
+                    metadata[metadataLine[0]] = metadataLine[1]
+
+    # get the content
+    content = ""
+    if "__TEXT__" in memberNames:
+        memberNames.remove("__TEXT__")
+
+        contentMember = tarFile.getmember("__TEXT__")
+        if not contentMember.issym() and contentMember.isfile():
+            content = _text_wrapper(tarFile.extractfile(contentMember)).read()
+
+    # get the remaining files as attachments
+    attachments = {}
+    for attachment in memberNames:
+        attachmentMember = tarFile.getmember(attachment)
+        if not attachmentMember.issym() and attachmentMember.isfile():
+            attachments[attachment] = tarFile.extractfile(attachmentMember).read()
+
+    parsed["content"] = content
+    parsed["metadata"] = metadata
+    parsed["attachments"] = attachments
+
+    return parsed