Skip to content

Commit fee8f87

Browse files
xiafu-msftxiafu-msft
authored
[Storage][Logging]Let users opt in logging_request_body (Azure#19710)
* [Storage][Logging]Let users opt in logging_request_body * Update policies.py * Update policies.py * Update policies.py * Update policies.py * logging response body * Update settings_fake.py
1 parent 524fd74 commit fee8f87

File tree

12 files changed

+563
-47
lines changed

12 files changed

+563
-47
lines changed

‎sdk/storage/azure-storage-blob/README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -345,6 +345,8 @@ Other optional configuration keyword arguments that can be specified on the clie
345345
* __user_agent__ (str): Appends the custom value to the user-agent header to be sent with the request.
346346
* __logging_enable__ (bool): Enables logging at the DEBUG level. Defaults to False. Can also be passed in at
347347
the client level to enable it for all requests.
348+
* __logging_body__ (bool): Enables logging the request and response body. Defaults to False. Can also be passed in at
349+
the client level to enable it for all requests.
348350
* __headers__ (dict): Pass in custom headers as key, value pairs. E.g. `headers={'CustomValue': value}`
349351

350352
## Troubleshooting

‎sdk/storage/azure-storage-blob/azure/storage/blob/_shared/policies.py

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
from io import SEEK_SET, UnsupportedOperation
1313
import logging
1414
import uuid
15-
import types
1615
from typing import Any, TYPE_CHECKING
1716
from wsgiref.handlers import format_date_time
1817
try:
@@ -183,11 +182,15 @@ class StorageLoggingPolicy(NetworkTraceLoggingPolicy):
183182
184183
This accepts both global configuration, and per-request level with "enable_http_logger"
185184
"""
185+
def __init__(self, logging_enable=False, **kwargs):
186+
self.logging_body = kwargs.pop("logging_body", False)
187+
super(StorageLoggingPolicy, self).__init__(logging_enable=logging_enable, **kwargs)
186188

187189
def on_request(self, request):
188190
# type: (PipelineRequest, Any) -> None
189191
http_request = request.http_request
190192
options = request.context.options
193+
self.logging_body = self.logging_body or options.pop("logging_body", False)
191194
if options.pop("logging_enable", self.enable_http_logger):
192195
request.context["logging_enable"] = True
193196
if not _LOGGER.isEnabledFor(logging.DEBUG):
@@ -216,11 +219,11 @@ def on_request(self, request):
216219
_LOGGER.debug(" %r: %r", header, value)
217220
_LOGGER.debug("Request body:")
218221

219-
# We don't want to log the binary data of a file upload.
220-
if isinstance(http_request.body, types.GeneratorType):
221-
_LOGGER.debug("File upload")
222-
else:
222+
if self.logging_body:
223223
_LOGGER.debug(str(http_request.body))
224+
else:
225+
# We don't want to log the binary data of a file upload.
226+
_LOGGER.debug("Hidden body, please use logging_body to show body")
224227
except Exception as err: # pylint: disable=broad-except
225228
_LOGGER.debug("Failed to log request: %r", err)
226229

@@ -240,19 +243,24 @@ def on_response(self, request, response):
240243
_LOGGER.debug("Response content:")
241244
pattern = re.compile(r'attachment; ?filename=["\w.]+', re.IGNORECASE)
242245
header = response.http_response.headers.get('content-disposition')
246+
resp_content_type = response.http_response.headers.get("content-type", "")
243247

244248
if header and pattern.match(header):
245249
filename = header.partition('=')[2]
246250
_LOGGER.debug("File attachments: %s", filename)
247-
elif response.http_response.headers.get("content-type", "").endswith("octet-stream"):
251+
elif resp_content_type.endswith("octet-stream"):
248252
_LOGGER.debug("Body contains binary data.")
249-
elif response.http_response.headers.get("content-type", "").startswith("image"):
253+
elif resp_content_type.startswith("image"):
250254
_LOGGER.debug("Body contains image data.")
251-
else:
252-
if response.context.options.get('stream', False):
255+
256+
if self.logging_body and resp_content_type.startswith("text"):
257+
_LOGGER.debug(response.http_response.text())
258+
elif self.logging_body:
259+
try:
260+
_LOGGER.debug(response.http_response.body())
261+
except ValueError:
253262
_LOGGER.debug("Body is streamable")
254-
else:
255-
_LOGGER.debug(response.http_response.text())
263+
256264
except Exception as err: # pylint: disable=broad-except
257265
_LOGGER.debug("Failed to log response: %s", repr(err))
258266

‎sdk/storage/azure-storage-blob/tests/recordings/test_logging.test_logging_request_and_response_body.yaml

Lines changed: 253 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,253 @@
1+
interactions:
2+
- request:
3+
body: null
4+
headers:
5+
Accept:
6+
- application/xml
7+
Accept-Encoding:
8+
- gzip, deflate
9+
Connection:
10+
- keep-alive
11+
Content-Length:
12+
- '0'
13+
User-Agent:
14+
- azsdk-python-storage-blob/12.9.0b1 Python/3.7.3 (Windows-10-10.0.19041-SP0)
15+
x-ms-date:
16+
- Thu, 22 Jul 2021 00:58:15 GMT
17+
x-ms-version:
18+
- '2020-08-04'
19+
method: PUT
20+
uri: https://storagename.blob.core.windows.net/utcontainer21d61510?restype=container
21+
response:
22+
body:
23+
string: "\uFEFF<?xml version=\"1.0\" encoding=\"utf-8\"?><Error><Code>ContainerAlreadyExists</Code><Message>The
24+
specified container already exists.\nRequestId:e1cc4389-801e-00bf-1f94-7e6451000000\nTime:2021-07-22T00:58:15.9669124Z</Message></Error>"
25+
headers:
26+
content-length:
27+
- '230'
28+
content-type:
29+
- application/xml
30+
date:
31+
- Thu, 22 Jul 2021 00:58:15 GMT
32+
server:
33+
- Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
34+
x-ms-error-code:
35+
- ContainerAlreadyExists
36+
x-ms-version:
37+
- '2020-08-04'
38+
status:
39+
code: 409
40+
message: The specified container already exists.
41+
- request:
42+
body: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
43+
headers:
44+
Accept:
45+
- application/xml
46+
Accept-Encoding:
47+
- gzip, deflate
48+
Connection:
49+
- keep-alive
50+
Content-Length:
51+
- '4096'
52+
Content-Type:
53+
- application/octet-stream
54+
User-Agent:
55+
- azsdk-python-storage-blob/12.9.0b1 Python/3.7.3 (Windows-10-10.0.19041-SP0)
56+
x-ms-blob-type:
57+
- BlockBlob
58+
x-ms-date:
59+
- Thu, 22 Jul 2021 00:58:16 GMT
60+
x-ms-version:
61+
- '2020-08-04'
62+
method: PUT
63+
uri: https://storagename.blob.core.windows.net/utcontainer21d61510/srcblob21d61510
64+
response:
65+
body:
66+
string: ''
67+
headers:
68+
content-length:
69+
- '0'
70+
content-md5:
71+
- IaGZxT9CKjgOILFi+26+nA==
72+
date:
73+
- Thu, 22 Jul 2021 00:58:15 GMT
74+
etag:
75+
- '"0x8D94CABC9EBBC01"'
76+
last-modified:
77+
- Thu, 22 Jul 2021 00:58:16 GMT
78+
server:
79+
- Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
80+
x-ms-content-crc64:
81+
- Ep3PX5ZZvPI=
82+
x-ms-request-server-encrypted:
83+
- 'true'
84+
x-ms-version:
85+
- '2020-08-04'
86+
status:
87+
code: 201
88+
message: Created
89+
- request:
90+
body: testloggingbody
91+
headers:
92+
Accept:
93+
- application/xml
94+
Accept-Encoding:
95+
- gzip, deflate
96+
Connection:
97+
- keep-alive
98+
Content-Length:
99+
- '15'
100+
Content-Type:
101+
- application/octet-stream
102+
User-Agent:
103+
- azsdk-python-storage-blob/12.9.0b1 Python/3.7.3 (Windows-10-10.0.19041-SP0)
104+
x-ms-blob-type:
105+
- BlockBlob
106+
x-ms-date:
107+
- Thu, 22 Jul 2021 00:58:16 GMT
108+
x-ms-version:
109+
- '2020-08-04'
110+
method: PUT
111+
uri: https://storagename.blob.core.windows.net/utcontainer21d61510/testloggingblob21d61510
112+
response:
113+
body:
114+
string: ''
115+
headers:
116+
content-length:
117+
- '0'
118+
content-md5:
119+
- kiWDMg2eTMJ7rcIxBGHjdQ==
120+
date:
121+
- Thu, 22 Jul 2021 00:58:15 GMT
122+
etag:
123+
- '"0x8D94CABC9FF6EAB"'
124+
last-modified:
125+
- Thu, 22 Jul 2021 00:58:16 GMT
126+
server:
127+
- Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
128+
x-ms-content-crc64:
129+
- V9QvlmP+jQ4=
130+
x-ms-request-server-encrypted:
131+
- 'true'
132+
x-ms-version:
133+
- '2020-08-04'
134+
status:
135+
code: 201
136+
message: Created
137+
- request:
138+
body: null
139+
headers:
140+
Accept:
141+
- application/xml
142+
Accept-Encoding:
143+
- gzip, deflate
144+
Connection:
145+
- keep-alive
146+
User-Agent:
147+
- azsdk-python-storage-blob/12.9.0b1 Python/3.7.3 (Windows-10-10.0.19041-SP0)
148+
x-ms-date:
149+
- Thu, 22 Jul 2021 00:58:16 GMT
150+
x-ms-range:
151+
- bytes=0-33554431
152+
x-ms-version:
153+
- '2020-08-04'
154+
method: GET
155+
uri: https://storagename.blob.core.windows.net/utcontainer21d61510/testloggingblob21d61510
156+
response:
157+
body:
158+
string: testloggingbody
159+
headers:
160+
accept-ranges:
161+
- bytes
162+
content-length:
163+
- '15'
164+
content-range:
165+
- bytes 0-14/15
166+
content-type:
167+
- application/octet-stream
168+
date:
169+
- Thu, 22 Jul 2021 00:58:15 GMT
170+
etag:
171+
- '"0x8D94CABC9FF6EAB"'
172+
last-modified:
173+
- Thu, 22 Jul 2021 00:58:16 GMT
174+
server:
175+
- Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
176+
vary:
177+
- Origin
178+
x-ms-blob-content-md5:
179+
- kiWDMg2eTMJ7rcIxBGHjdQ==
180+
x-ms-blob-type:
181+
- BlockBlob
182+
x-ms-creation-time:
183+
- Thu, 22 Jul 2021 00:18:58 GMT
184+
x-ms-lease-state:
185+
- available
186+
x-ms-lease-status:
187+
- unlocked
188+
x-ms-server-encrypted:
189+
- 'true'
190+
x-ms-version:
191+
- '2020-08-04'
192+
status:
193+
code: 206
194+
message: Partial Content
195+
- request:
196+
body: null
197+
headers:
198+
Accept:
199+
- application/xml
200+
Accept-Encoding:
201+
- gzip, deflate
202+
Connection:
203+
- keep-alive
204+
User-Agent:
205+
- azsdk-python-storage-blob/12.9.0b1 Python/3.7.3 (Windows-10-10.0.19041-SP0)
206+
x-ms-date:
207+
- Thu, 22 Jul 2021 00:58:16 GMT
208+
x-ms-range:
209+
- bytes=0-33554431
210+
x-ms-version:
211+
- '2020-08-04'
212+
method: GET
213+
uri: https://storagename.blob.core.windows.net/utcontainer21d61510/testloggingblob21d61510
214+
response:
215+
body:
216+
string: testloggingbody
217+
headers:
218+
accept-ranges:
219+
- bytes
220+
content-length:
221+
- '15'
222+
content-range:
223+
- bytes 0-14/15
224+
content-type:
225+
- application/octet-stream
226+
date:
227+
- Thu, 22 Jul 2021 00:58:15 GMT
228+
etag:
229+
- '"0x8D94CABC9FF6EAB"'
230+
last-modified:
231+
- Thu, 22 Jul 2021 00:58:16 GMT
232+
server:
233+
- Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
234+
vary:
235+
- Origin
236+
x-ms-blob-content-md5:
237+
- kiWDMg2eTMJ7rcIxBGHjdQ==
238+
x-ms-blob-type:
239+
- BlockBlob
240+
x-ms-creation-time:
241+
- Thu, 22 Jul 2021 00:18:58 GMT
242+
x-ms-lease-state:
243+
- available
244+
x-ms-lease-status:
245+
- unlocked
246+
x-ms-server-encrypted:
247+
- 'true'
248+
x-ms-version:
249+
- '2020-08-04'
250+
status:
251+
code: 206
252+
message: Partial Content
253+
version: 1

0 commit comments

Comments
 (0)