Skip to content

Commit 25179e5

Browse files
CitoCito
committed
Fix processing binary data in FieldStorage (#14)
1 parent 526b8e3 commit 25179e5

File tree

3 files changed

+108
-11
lines changed

3 files changed

+108
-11
lines changed

‎webware/HTTPRequest.py

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ def __init__(self, requestDict=None):
3535
self._requestID = requestDict['requestID']
3636
# Protect the loading of fields with an exception handler,
3737
# because bad headers sometimes can break the field storage
38-
# (see also https://bugs.python.org/issue27777).
38+
# (see also https://github.com/python/cpython/issues/71964).
3939
try:
4040
self._fields = FieldStorage(
4141
self._input, environ=self._environ,
@@ -47,7 +47,7 @@ def __init__(self, requestDict=None):
4747
if 'HTTP_COOKIE' in self._environ:
4848
# If there are duplicate cookies, always use the first one
4949
# because it is the most relevant one according to RFC 2965
50-
# (workaround for https://bugs.python.org/issue1375011).
50+
# (see also https://github.com/python/cpython/issues/42664).
5151
# noinspection PyTypeChecker
5252
cookies = dict(cookie.split('=', 1) for cookie in reversed(
5353
self._environ['HTTP_COOKIE'].split('; ')))
@@ -321,15 +321,15 @@ def urlPath(self):
321321
322322
This is actually the same as pathInfo().
323323
324-
For example, http://host/Webware/Context/Servlet?x=1
324+
For example, https://host/Webware/Context/Servlet?x=1
325325
yields '/Context/Servlet'.
326326
"""
327327
return self._pathInfo
328328

329329
def urlPathDir(self):
330330
"""Same as urlPath, but only gives the directory.
331331
332-
For example, http://host/Webware/Context/Servlet?x=1
332+
For example, https://host/Webware/Context/Servlet?x=1
333333
yields '/Context'.
334334
"""
335335
return os.path.dirname(self.urlPath())
@@ -434,7 +434,7 @@ def serverURL(self, canonical=False):
434434
then the canonical hostname of the server is used if possible.
435435
436436
The path is returned without any extra path info or query strings,
437-
i.e. http://www.my.own.host.com:8080/Webware/TestPage.py
437+
i.e. https://www.my.own.host.com:8080/Webware/TestPage.py
438438
"""
439439
if canonical and 'SCRIPT_URI' in self._environ:
440440
return self._environ['SCRIPT_URI']
@@ -801,7 +801,8 @@ def info(self):
801801
]
802802
for method in _infoMethods:
803803
try:
804-
info.append((method.__name__, method(self)))
804+
# noinspection PyArgumentList
805+
info.append((method.__name__, method(self),))
805806
except Exception:
806807
info.append((method.__name__, None))
807808
return info

‎webware/WebUtils/FieldStorage.py

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -416,7 +416,6 @@ def read_single(self):
416416

417417
def read_binary(self):
418418
"""Internal: read binary data."""
419-
self.file = self.make_file()
420419
todo = self.length
421420
while todo > 0:
422421
data = self.fp.read(min(todo, self.bufsize))
@@ -428,10 +427,14 @@ def read_binary(self):
428427
if not data:
429428
self.done = -1
430429
break
431-
if self._binary_file:
432-
self.file.write(data)
433-
else: # fix for issue 27777
434-
self.file.write(data.decode())
430+
if not self._binary_file:
431+
# fix for https://github.com/python/cpython/pull/7804
432+
try:
433+
data = data.decode()
434+
except UnicodeDecodeError:
435+
self._binary_file = True
436+
self.file = self.make_file()
437+
self.file.write(data)
435438
todo -= len(data)
436439

437440
def read_lines(self):

‎webware/WebUtils/Tests/TestFieldStorageModified.py

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,3 +131,96 @@ def testPostRequestWithQueryWithSemicolon2(self):
131131
self.assertEqual(fs.getlist('a'), ['1'])
132132
self.assertEqual(fs.getlist('b'), ['2', '3'])
133133
self.assertEqual(fs.getlist('c'), ['3'])
134+
135+
def testPostRequestWithoutContentLength(self):
136+
# see https://github.com/python/cpython/issues/71964
137+
fs = FieldStorage(
138+
fp=BytesIO(b'{"test":123}'),
139+
environ={'REQUEST_METHOD': 'POST',
140+
'CONTENT_TYPE': 'application/json'})
141+
self.assertEqual(fs.headers, {
142+
'content-type': 'application/json'})
143+
self.assertEqual(fs.type, 'application/json')
144+
self.assertEqual(fs.length, -1)
145+
self.assertEqual(fs.bytes_read, 12)
146+
assert fs.file.read() == '{"test":123}'
147+
148+
def testPostRequestWithContentLengthAndContentDispositionInline(self):
149+
# see https://github.com/python/cpython/issues/71964
150+
fs = FieldStorage(
151+
fp=BytesIO(b'{"test":123}'),
152+
headers={'content-length': 12, 'content-disposition': 'inline',
153+
'content-type': 'application/json'},
154+
environ={'REQUEST_METHOD': 'POST'})
155+
self.assertEqual(fs.headers, {
156+
'content-type': 'application/json', 'content-length': 12,
157+
'content-disposition': 'inline'})
158+
self.assertEqual(fs.disposition, 'inline')
159+
self.assertIsNone(fs.filename)
160+
self.assertEqual(fs.type, 'application/json')
161+
self.assertEqual(fs.length, 12)
162+
self.assertEqual(fs.bytes_read, 12)
163+
self.assertEqual(fs.file.read(), '{"test":123}')
164+
165+
def testPostRequestWithContentLengthAndContentDispositionAttachment(self):
166+
# not affected by https://github.com/python/cpython/issues/71964
167+
fs = FieldStorage(
168+
fp=BytesIO(b'{"test":123}'),
169+
headers={'content-length': 12,
170+
'content-disposition': 'attachment; filename="foo.json"',
171+
'content-type': 'application/json'},
172+
environ={'REQUEST_METHOD': 'POST'})
173+
self.assertEqual(fs.headers, {
174+
'content-type': 'application/json', 'content-length': 12,
175+
'content-disposition': 'attachment; filename="foo.json"'})
176+
self.assertEqual(fs.disposition, 'attachment')
177+
self.assertEqual(fs.filename, 'foo.json')
178+
self.assertEqual(fs.type, 'application/json')
179+
self.assertEqual(fs.length, 12)
180+
self.assertEqual(fs.bytes_read, 12)
181+
self.assertEqual(fs.file.read(), b'{"test":123}')
182+
183+
def testPostRequestWithContentLengthButWithoutContentDisposition(self):
184+
# see https://github.com/python/cpython/issues/71964
185+
fs = FieldStorage(fp=BytesIO(b'{"test":123}'), environ={
186+
'CONTENT_LENGTH': 12, 'REQUEST_METHOD': 'POST',
187+
'CONTENT_TYPE': 'application/json'})
188+
self.assertEqual(fs.headers, {
189+
'content-type': 'application/json', 'content-length': 12})
190+
self.assertEqual(fs.disposition, '')
191+
self.assertEqual(fs.type, 'application/json')
192+
self.assertEqual(fs.length, 12)
193+
self.assertEqual(fs.bytes_read, 12)
194+
self.assertEqual(fs.file.read(), '{"test":123}')
195+
196+
def testPostRequestWithUtf8BinaryData(self):
197+
text = 'The \u2603 by Raymond Briggs'
198+
content = text.encode('utf-8')
199+
length = len(content)
200+
fs = FieldStorage(fp=BytesIO(content), environ={
201+
'CONTENT_LENGTH': length, 'REQUEST_METHOD': 'POST',
202+
'CONTENT_TYPE': 'application/octet-stream'})
203+
self.assertEqual(fs.headers, {
204+
'content-type': 'application/octet-stream',
205+
'content-length': length})
206+
self.assertEqual(fs.type, 'application/octet-stream')
207+
self.assertEqual(fs.length, length)
208+
self.assertEqual(fs.bytes_read, length)
209+
self.assertEqual(fs.file.read(), text)
210+
211+
def testPostRequestWithNonUtf8BinaryData(self):
212+
# see https://github.com/WebwareForPython/w4py3/issues/14
213+
content = b'\xfe\xff\xc0'
214+
with self.assertRaises(UnicodeDecodeError):
215+
content.decode('utf-8')
216+
length = len(content)
217+
fs = FieldStorage(fp=BytesIO(content), environ={
218+
'CONTENT_LENGTH': length, 'REQUEST_METHOD': 'POST',
219+
'CONTENT_TYPE': 'application/octet-stream'})
220+
self.assertEqual(fs.headers, {
221+
'content-type': 'application/octet-stream',
222+
'content-length': length})
223+
self.assertEqual(fs.type, 'application/octet-stream')
224+
self.assertEqual(fs.length, length)
225+
self.assertEqual(fs.bytes_read, length)
226+
self.assertEqual(fs.file.read(), content)

0 commit comments

Comments
 (0)