Merge pull request #259 from jjelosua/master

expose service in the from_file parser

Author: chrismattmann

tika/parser.py

@@ -20,7 +20,7 @@
 import os
 import json
 
-def from_file(filename, serverEndpoint=ServerEndpoint, xmlContent=False, headers=None, config_path=None, requestOptions={}):
+def from_file(filename, service='all', serverEndpoint=ServerEndpoint, xmlContent=False, headers=None, config_path=None, requestOptions={}):
     '''
     Parses a file for metadata and content
     :param filename: path to file which needs to be parsed
@@ -33,11 +33,11 @@ def from_file(filename, serverEndpoint=ServerEndpoint, xmlContent=False, headers
             'content' has a str value and metadata has a dict type value.
     '''
     if not xmlContent:
-        jsonOutput = parse1('all', filename, serverEndpoint, headers=headers, config_path=config_path, requestOptions=requestOptions)
+        output = parse1(service, filename, serverEndpoint, headers=headers, config_path=config_path, requestOptions=requestOptions)
     else:
-        jsonOutput = parse1('all', filename, serverEndpoint, services={'meta': '/meta', 'text': '/tika', 'all': '/rmeta/xml'},
+        output = parse1(service, filename, serverEndpoint, services={'meta': '/meta', 'text': '/tika', 'all': '/rmeta/xml'},
                             headers=headers, config_path=config_path, requestOptions=requestOptions)
-    return _parse(jsonOutput)
+    return _parse(output, service)
 
 
 def from_buffer(string, serverEndpoint=ServerEndpoint, xmlContent=False, headers=None, config_path=None, requestOptions={}):
@@ -61,20 +61,35 @@ def from_buffer(string, serverEndpoint=ServerEndpoint, xmlContent=False, headers
 
     return _parse((status,response))
 
-def _parse(jsonOutput):
+def _parse(output, service='all'):
     '''
-    Parses JSON response from Tika REST API server
-    :param jsonOutput: JSON output from Tika Server
+    Parses response from Tika REST API server
+    :param output: output from Tika Server
+    :param service: service requested from the tika server
+                    Default is 'all', which results in recursive text content+metadata.
+                    'meta' returns only metadata
+                    'text' returns only content
     :return: a dictionary having 'metadata' and 'content' values
     '''
-    parsed={}
-    if not jsonOutput:
+    parsed={'metadata': None, 'content': None}
+    if not output:
         return parsed
-    
-    parsed["status"] = jsonOutput[0]
-    if jsonOutput[1] == None or jsonOutput[1] == "":
+
+    parsed["status"] = output[0]
+    if output[1] == None or output[1] == "":
+        return parsed
+
+    if service == "text":
+        parsed["content"] = output[1]
+        return parsed
+
+    realJson = json.loads(output[1])
+
+    parsed["metadata"] = {}
+    if service == "meta":
+        for key in realJson:
+            parsed["metadata"][key] = realJson[key]
         return parsed
-    realJson = json.loads(jsonOutput[1])
 
     content = ""
     for js in realJson:
@@ -85,7 +100,6 @@ def _parse(jsonOutput):
         content = None
 
     parsed["content"] = content
-    parsed["metadata"] = {}
 
     for js in realJson:
         for n in js:

tika/tests/test_from_file_service.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+# encoding: utf-8
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# python -m unittest tika.tests.test_from_file_service
+
+import unittest
+import tika.parser
+
+
+class CreateTest(unittest.TestCase):
+    'test different services in from_file parsing: Content, Metadata or both in recursive mode'
+
+    def test_default_service(self):
+        'parse file using default service'
+        result = tika.parser.from_file(
+            'https://boe.es/boe/dias/2019/12/02/pdfs/BOE-A-2019-17288.pdf')
+        self.assertEqual(result['metadata']['Content-Type'],'application/pdf')
+        self.assertIn('AUTORIDADES Y PERSONAL',result['content'])
+    def test_default_service_explicit(self):
+        'parse file using default service explicitly'
+        result = tika.parser.from_file(
+            'https://boe.es/boe/dias/2019/12/02/pdfs/BOE-A-2019-17288.pdf', service='all')
+        self.assertEqual(result['metadata']['Content-Type'],'application/pdf')
+        self.assertIn('AUTORIDADES Y PERSONAL',result['content'])
+    def test_text_service(self):
+        'parse file using the content only service'
+        result = tika.parser.from_file(
+            'https://boe.es/boe/dias/2019/12/02/pdfs/BOE-A-2019-17288.pdf', service='text')
+        self.assertIsNone(result['metadata'])
+        self.assertIn('AUTORIDADES Y PERSONAL',result['content'])
+    def test_meta_service(self):
+        'parse file using the content only service'
+        result = tika.parser.from_file(
+            'https://boe.es/boe/dias/2019/12/02/pdfs/BOE-A-2019-17288.pdf', service='meta')
+        self.assertIsNone(result['content'])
+        self.assertEqual(result['metadata']['Content-Type'],'application/pdf')
+    def test_invalid_service(self):
+        'parse file using an invalid service should perform the default parsing'
+        result = tika.parser.from_file(
+            'https://boe.es/boe/dias/2019/12/02/pdfs/BOE-A-2019-17288.pdf', service='bad')
+        self.assertEqual(result['metadata']['Content-Type'],'application/pdf')
+        self.assertIn('AUTORIDADES Y PERSONAL',result['content'])
+
+if __name__ == '__main__':
+    unittest.main()