Use content type in FieldStorage (#14)

Author: Cito

webware/WebUtils/FieldStorage.py

@@ -187,6 +187,8 @@ def __init__(self, fp=None, headers=None, outerboundary=b'',
             ctype, pdict = 'application/x-www-form-urlencoded', {}
         self.type = ctype
         self.type_options = pdict
+        if not self._binary_file and isBinaryType(ctype, pdict):
+            self._binary_file = True
         self.innerboundary = pdict['boundary'].encode(
             self.encoding, self.errors) if 'boundary' in pdict else b''
 
@@ -637,3 +639,13 @@ def hasSeparator():
     except TypeError:  # Python < 3.9.2
         return False
     return True
+
+
+def isBinaryType(ctype, pdict=None):
+    """"Check whether the given MIME type uses binary data."""
+    if pdict and pdict.get('charset') == 'binary':
+        return True
+    return not (
+        ctype.startswith('text/') or ctype.endswith(('+json', '+xml')) or
+        (ctype.startswith('application') and
+         ctype.endswith(('/json', '/xml', '/ecmascript', '/javascript'))))

webware/WebUtils/Tests/TestFieldStorageModified.py

@@ -4,7 +4,7 @@
 
 from io import BytesIO
 
-from WebUtils.FieldStorage import FieldStorage, hasSeparator
+from WebUtils.FieldStorage import FieldStorage, hasSeparator, isBinaryType
 
 
 class TestFieldStorage(unittest.TestCase):
@@ -206,7 +206,7 @@ def testPostRequestWithUtf8BinaryData(self):
         self.assertEqual(fs.type, 'application/octet-stream')
         self.assertEqual(fs.length, length)
         self.assertEqual(fs.bytes_read, length)
-        self.assertEqual(fs.file.read(), text)
+        self.assertEqual(fs.file.read(), content)
 
     def testPostRequestWithNonUtf8BinaryData(self):
         # see https://github.com/WebwareForPython/w4py3/issues/14
@@ -224,3 +224,57 @@ def testPostRequestWithNonUtf8BinaryData(self):
         self.assertEqual(fs.length, length)
         self.assertEqual(fs.bytes_read, length)
         self.assertEqual(fs.file.read(), content)
+
+    def testPostRequestWithUtf8TextData(self):
+        text = 'The \u2603 by Raymond Briggs'
+        content = text.encode('utf-8')
+        length = len(content)
+        fs = FieldStorage(fp=BytesIO(content), environ={
+            'CONTENT_LENGTH': length, 'REQUEST_METHOD': 'POST',
+            'CONTENT_TYPE': 'text/plain'})
+        self.assertEqual(fs.headers, {
+            'content-type': 'text/plain',
+            'content-length': length})
+        self.assertEqual(fs.type, 'text/plain')
+        self.assertEqual(fs.length, length)
+        self.assertEqual(fs.bytes_read, length)
+        self.assertEqual(fs.file.read(), text)
+
+    def testPostRequestWithNonUtf8TextData(self):
+        # see https://github.com/WebwareForPython/w4py3/issues/14
+        content = b'\xfe\xff\xc0'
+        with self.assertRaises(UnicodeDecodeError):
+            content.decode('utf-8')
+        length = len(content)
+        fs = FieldStorage(fp=BytesIO(content), environ={
+            'CONTENT_LENGTH': length, 'REQUEST_METHOD': 'POST',
+            'CONTENT_TYPE': 'text/plain'})
+        self.assertEqual(fs.headers, {
+            'content-type': 'text/plain',
+            'content-length': length})
+        self.assertEqual(fs.type, 'text/plain')
+        self.assertEqual(fs.length, length)
+        self.assertEqual(fs.bytes_read, length)
+        self.assertEqual(fs.file.read(), content)
+
+    def testIsBinaryType(self):
+        self.assertIs(isBinaryType('application/json'), False)
+        self.assertIs(isBinaryType('application/xml'), False)
+        self.assertIs(isBinaryType('application/calendar+json'), False)
+        self.assertIs(isBinaryType('application/calendar+xml'), False)
+        self.assertIs(isBinaryType('model/x3d+xml'), False)
+        self.assertIs(isBinaryType('text/csv'), False)
+        self.assertIs(isBinaryType('text/html'), False)
+        self.assertIs(isBinaryType('text/plain'), False)
+        self.assertIs(isBinaryType('x3d+xml'), False)
+        self.assertIs(isBinaryType('application/octet-stream'), True)
+        self.assertIs(isBinaryType('application/pdf'), True)
+        self.assertIs(isBinaryType('application/zip'), True)
+        self.assertIs(isBinaryType('audio/ogg'), True)
+        self.assertIs(isBinaryType('font/otf'), True)
+        self.assertIs(isBinaryType('image/png'), True)
+        self.assertIs(isBinaryType('video/mp4'), True)
+        self.assertIs(isBinaryType('application/json',
+                                   {'charset': 'utf8'}), False)
+        self.assertIs(isBinaryType('text/csv',
+                                   {'charset': 'binary'}), True)