Skip to content

Commit ed0ddab

Browse files
author
Andrey Slotin
authored
Instrument Google Cloud Storage client (#269)
* Log requests to the Google Cloud Storage API made with google.cloud.storage * Collect GCS API bucket operation tags * Collect GCS API blob operation tags * Collect GCS API channel operation tags * Collect GCS API default object ACL operation tags * Collect GCS API object ACL operation tags * Collect GCS API HMAC keys operation tags * Collect GCS API service account operation tags * Auto-instrument google-cloud-storage client * Instrument google-cloud-storage for Python 3 only The library has dropped support for Python 2 * Collect GCS batch operation tags * Move GCS API tag collectors into a separate file * Register GCS span format * Add trace context propagation test util * Add Google Cloud Storage instrumentation tests * Lower the min supported version of google-cloud-storage to 1.24.0
1 parent 76d3cc4 commit ed0ddab

File tree

8 files changed

+1438
-2
lines changed

8 files changed

+1438
-2
lines changed

‎instana/__init__.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,9 @@ def boot_agent():
137137
else:
138138
from .instrumentation import mysqlclient
139139

140+
if sys.version_info[0] >= 3:
141+
from .instrumentation.google.cloud import storage
142+
140143
from .instrumentation.celery import hooks
141144

142145
from .instrumentation import cassandra_inst

‎instana/instrumentation/google/cloud/collectors.py‎

Lines changed: 317 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,317 @@
1+
import re
2+
3+
try:
4+
# Python 3
5+
from urllib.parse import unquote
6+
except ImportError:
7+
# Python 2
8+
from urllib import unquote
9+
10+
# _storage_api defines a conversion of Google Storage JSON API requests into span tags as follows:
11+
# request_method -> path_matcher -> collector
12+
#
13+
# * request method - the HTTP method used to make an API request (GET, POST, etc.)
14+
# * path_matcher - either a string or a regex applied to the API request path (string values match first).
15+
# * collector - a lambda returning a dict of span from API request query string.
16+
# parameters and request body data. If a regex is used as a path matcher, the match result
17+
# will be provided as a third argument.
18+
#
19+
# The API documentation can be found at https://cloud.google.com/storage/docs/json_api
20+
_storage_api = {
21+
'GET': {
22+
#####################
23+
# Bucket operations #
24+
#####################
25+
'/b': lambda params, data: {
26+
'gcs.op': 'buckets.list',
27+
'gcs.projectId': params.get('project', None)
28+
},
29+
re.compile('^/b/(?P<bucket>[^/]+)$'): lambda params, data, match: {
30+
'gcs.op': 'buckets.get',
31+
'gcs.bucket': unquote(match.group('bucket')),
32+
},
33+
re.compile('^/b/(?P<bucket>[^/]+)/iam$'): lambda params, data, match: {
34+
'gcs.op': 'buckets.getIamPolicy',
35+
'gcs.bucket': unquote(match.group('bucket')),
36+
},
37+
re.compile('^/b/(?P<bucket>[^/]+)/iam/testPermissions$'): lambda params, data, match: {
38+
'gcs.op': 'buckets.testIamPermissions',
39+
'gcs.bucket': unquote(match.group('bucket')),
40+
},
41+
42+
##########################
43+
# Object/blob operations #
44+
##########################
45+
re.compile('^/b/(?P<bucket>[^/]+)/o/(?P<object>[^/]+)$'): lambda params, data, match: {
46+
'gcs.op': params.get('alt', 'json') == 'media' and 'objects.get' or 'objects.attrs',
47+
'gcs.bucket': unquote(match.group('bucket')),
48+
'gcs.object': unquote(match.group('object')),
49+
},
50+
re.compile('^/b/(?P<bucket>[^/]+)/o$'): lambda params, data, match: {
51+
'gcs.op': 'objects.list',
52+
'gcs.bucket': unquote(match.group('bucket'))
53+
},
54+
55+
##################################
56+
# Default object ACLs operations #
57+
##################################
58+
re.compile('^/b/(?P<bucket>[^/]+)/defaultObjectAcl/(?P<entity>[^/]+)$'): lambda params, data, match: {
59+
'gcs.op': 'defaultAcls.get',
60+
'gcs.bucket': unquote(match.group('bucket')),
61+
'gcs.entity': unquote(match.group('entity'))
62+
},
63+
re.compile('^/b/(?P<bucket>[^/]+)/defaultObjectAcl$'): lambda params, data, match: {
64+
'gcs.op': 'defaultAcls.list',
65+
'gcs.bucket': unquote(match.group('bucket')),
66+
},
67+
68+
#########################
69+
# Object ACL operations #
70+
#########################
71+
re.compile('^/b/(?P<bucket>[^/]+)/o/(?P<object>[^/]+)/acl/(?P<entity>[^/]+)$'): lambda params, data, match: {
72+
'gcs.op': 'objectAcls.get',
73+
'gcs.bucket': unquote(match.group('bucket')),
74+
'gcs.object': unquote(match.group('object')),
75+
'gcs.entity': unquote(match.group('entity'))
76+
},
77+
re.compile('^/b/(?P<bucket>[^/]+)/o/(?P<object>[^/]+)/acl$'): lambda params, data, match: {
78+
'gcs.op': 'objectAcls.list',
79+
'gcs.bucket': unquote(match.group('bucket')),
80+
'gcs.object': unquote(match.group('object'))
81+
},
82+
83+
########################
84+
# HMAC keys operations #
85+
########################
86+
re.compile('^/projects/(?P<project>[^/]+)/hmacKeys$'): lambda params, data, match: {
87+
'gcs.op': 'hmacKeys.list',
88+
'gcs.projectId': unquote(match.group('project'))
89+
},
90+
re.compile('^/projects/(?P<project>[^/]+)/hmacKeys/(?P<accessId>[^/]+)$'): lambda params, data, match: {
91+
'gcs.op': 'hmacKeys.get',
92+
'gcs.projectId': unquote(match.group('project')),
93+
'gcs.accessId': unquote(match.group('accessId'))
94+
},
95+
96+
##############################
97+
# Service account operations #
98+
##############################
99+
re.compile('^/projects/(?P<project>[^/]+)/serviceAccount$'): lambda params, data, match: {
100+
'gcs.op': 'serviceAccount.get',
101+
'gcs.projectId': unquote(match.group('project'))
102+
}
103+
},
104+
'POST': {
105+
#####################
106+
# Bucket operations #
107+
#####################
108+
'/b': lambda params, data: {
109+
'gcs.op': 'buckets.insert',
110+
'gcs.projectId': params.get('project', None),
111+
'gcs.bucket': data.get('name', None),
112+
},
113+
re.compile('^/b/(?P<bucket>[^/]+)/lockRetentionPolicy$'): lambda params, data, match: {
114+
'gcs.op': 'buckets.lockRetentionPolicy',
115+
'gcs.bucket': unquote(match.group('bucket')),
116+
},
117+
118+
##########################
119+
# Object/blob operations #
120+
##########################
121+
re.compile('^/b/(?P<bucket>[^/]+)/o/(?P<object>[^/]+)/compose$'): lambda params, data, match: {
122+
'gcs.op': 'objects.compose',
123+
'gcs.destinationBucket': unquote(match.group('bucket')),
124+
'gcs.destinationObject': unquote(match.group('object')),
125+
'gcs.sourceObjects': ','.join(
126+
['%s/%s' % (unquote(match.group('bucket')), o['name']) for o in data.get('sourceObjects', []) if 'name' in o]
127+
)
128+
},
129+
re.compile('^/b/(?P<srcBucket>[^/]+)/o/(?P<srcObject>[^/]+)/copyTo/b/(?P<destBucket>[^/]+)/o/(?P<destObject>[^/]+)$'): lambda params, data, match: {
130+
'gcs.op': 'objects.copy',
131+
'gcs.destinationBucket': unquote(match.group('destBucket')),
132+
'gcs.destinationObject': unquote(match.group('destObject')),
133+
'gcs.sourceBucket': unquote(match.group('srcBucket')),
134+
'gcs.sourceObject': unquote(match.group('srcObject')),
135+
},
136+
re.compile('^/b/(?P<bucket>[^/]+)/o$'): lambda params, data, match: {
137+
'gcs.op': 'objects.insert',
138+
'gcs.bucket': unquote(match.group('bucket')),
139+
'gcs.object': params.get('name', data.get('name', None)),
140+
},
141+
re.compile('^/b/(?P<srcBucket>[^/]+)/o/(?P<srcObject>[^/]+)/rewriteTo/b/(?P<destBucket>[^/]+)/o/(?P<destObject>[^/]+)$'): lambda params, data, match: {
142+
'gcs.op': 'objects.rewrite',
143+
'gcs.destinationBucket': unquote(match.group('destBucket')),
144+
'gcs.destinationObject': unquote(match.group('destObject')),
145+
'gcs.sourceBucket': unquote(match.group('srcBucket')),
146+
'gcs.sourceObject': unquote(match.group('srcObject')),
147+
},
148+
149+
######################
150+
# Channel operations #
151+
######################
152+
'/channels/stop': lambda params, data: {
153+
'gcs.op': 'channels.stop',
154+
'gcs.entity': data.get('id', None)
155+
},
156+
157+
##################################
158+
# Default object ACLs operations #
159+
##################################
160+
re.compile('^/b/(?P<bucket>[^/]+)/defaultObjectAcl$'): lambda params, data, match: {
161+
'gcs.op': 'defaultAcls.insert',
162+
'gcs.bucket': unquote(match.group('bucket')),
163+
'gcs.entity': data.get('entity', None)
164+
},
165+
166+
#########################
167+
# Object ACL operations #
168+
#########################
169+
re.compile('^/b/(?P<bucket>[^/]+)/o/(?P<object>[^/]+)/acl$'): lambda params, data, match: {
170+
'gcs.op': 'objectAcls.insert',
171+
'gcs.bucket': unquote(match.group('bucket')),
172+
'gcs.object': unquote(match.group('object')),
173+
'gcs.entity': data.get('entity', None)
174+
},
175+
176+
########################
177+
# HMAC keys operations #
178+
########################
179+
re.compile('^/projects/(?P<project>[^/]+)/hmacKeys$'): lambda params, data, match: {
180+
'gcs.op': 'hmacKeys.create',
181+
'gcs.projectId': unquote(match.group('project'))
182+
}
183+
},
184+
'PATCH': {
185+
#####################
186+
# Bucket operations #
187+
#####################
188+
re.compile('^/b/(?P<bucket>[^/]+)$'): lambda params, data, match: {
189+
'gcs.op': 'buckets.patch',
190+
'gcs.bucket': unquote(match.group('bucket')),
191+
},
192+
193+
##########################
194+
# Object/blob operations #
195+
##########################
196+
re.compile('^/b/(?P<bucket>[^/]+)/o/(?P<object>[^/]+)$'): lambda params, data, match: {
197+
'gcs.op': 'objects.patch',
198+
'gcs.bucket': unquote(match.group('bucket')),
199+
'gcs.object': unquote(match.group('object')),
200+
},
201+
202+
##################################
203+
# Default object ACLs operations #
204+
##################################
205+
re.compile('^/b/(?P<bucket>[^/]+)/defaultObjectAcl/(?P<entity>[^/]+)$'): lambda params, data, match: {
206+
'gcs.op': 'defaultAcls.patch',
207+
'gcs.bucket': unquote(match.group('bucket')),
208+
'gcs.entity': unquote(match.group('entity'))
209+
},
210+
211+
#########################
212+
# Object ACL operations #
213+
#########################
214+
re.compile('^/b/(?P<bucket>[^/]+)/o/(?P<object>[^/]+)/acl/(?P<entity>[^/]+)$'): lambda params, data, match: {
215+
'gcs.op': 'objectAcls.patch',
216+
'gcs.bucket': unquote(match.group('bucket')),
217+
'gcs.object': unquote(match.group('object')),
218+
'gcs.entity': unquote(match.group('entity'))
219+
}
220+
},
221+
'PUT': {
222+
#####################
223+
# Bucket operations #
224+
#####################
225+
re.compile('^/b/(?P<bucket>[^/]+)$'): lambda params, data, match: {
226+
'gcs.op': 'buckets.update',
227+
'gcs.bucket': unquote(match.group('bucket')),
228+
},
229+
re.compile('^/b/(?P<bucket>[^/]+)/iam$'): lambda params, data, match: {
230+
'gcs.op': 'buckets.setIamPolicy',
231+
'gcs.bucket': unquote(match.group('bucket')),
232+
},
233+
234+
##########################
235+
# Object/blob operations #
236+
##########################
237+
re.compile('^/b/(?P<bucket>[^/]+)/o/(?P<object>[^/]+)$'): lambda params, data, match: {
238+
'gcs.op': 'objects.update',
239+
'gcs.bucket': unquote(match.group('bucket')),
240+
'gcs.object': unquote(match.group('object')),
241+
},
242+
243+
##################################
244+
# Default object ACLs operations #
245+
##################################
246+
re.compile('^/b/(?P<bucket>[^/]+)/defaultObjectAcl/(?P<entity>[^/]+)$'): lambda params, data, match: {
247+
'gcs.op': 'defaultAcls.update',
248+
'gcs.bucket': unquote(match.group('bucket')),
249+
'gcs.entity': unquote(match.group('entity'))
250+
},
251+
252+
#########################
253+
# Object ACL operations #
254+
#########################
255+
re.compile('^/b/(?P<bucket>[^/]+)/o/(?P<object>[^/]+)/acl/(?P<entity>[^/]+)$'): lambda params, data, match: {
256+
'gcs.op': 'objectAcls.update',
257+
'gcs.bucket': unquote(match.group('bucket')),
258+
'gcs.object': unquote(match.group('object')),
259+
'gcs.entity': unquote(match.group('entity'))
260+
},
261+
262+
########################
263+
# HMAC keys operations #
264+
########################
265+
re.compile('^/projects/(?P<project>[^/]+)/hmacKeys/(?P<accessId>[^/]+)$'): lambda params, data, match: {
266+
'gcs.op': 'hmacKeys.update',
267+
'gcs.projectId': unquote(match.group('project')),
268+
'gcs.accessId': unquote(match.group('accessId'))
269+
}
270+
},
271+
'DELETE': {
272+
#####################
273+
# Bucket operations #
274+
#####################
275+
re.compile('^/b/(?P<bucket>[^/]+)$'): lambda params, data, match: {
276+
'gcs.op': 'buckets.delete',
277+
'gcs.bucket': unquote(match.group('bucket')),
278+
},
279+
280+
##########################
281+
# Object/blob operations #
282+
##########################
283+
re.compile('^/b/(?P<bucket>[^/]+)/o/(?P<object>[^/]+)$'): lambda params, data, match: {
284+
'gcs.op': 'objects.delete',
285+
'gcs.bucket': unquote(match.group('bucket')),
286+
'gcs.object': unquote(match.group('object')),
287+
},
288+
289+
##################################
290+
# Default object ACLs operations #
291+
##################################
292+
re.compile('^/b/(?P<bucket>[^/]+)/defaultObjectAcl/(?P<entity>[^/]+)$'): lambda params, data, match: {
293+
'gcs.op': 'defaultAcls.delete',
294+
'gcs.bucket': unquote(match.group('bucket')),
295+
'gcs.entity': unquote(match.group('entity'))
296+
},
297+
298+
#########################
299+
# Object ACL operations #
300+
#########################
301+
re.compile('^/b/(?P<bucket>[^/]+)/o/(?P<object>[^/]+)/acl/(?P<entity>[^/]+)$'): lambda params, data, match: {
302+
'gcs.op': 'objectAcls.delete',
303+
'gcs.bucket': unquote(match.group('bucket')),
304+
'gcs.object': unquote(match.group('object')),
305+
'gcs.entity': unquote(match.group('entity'))
306+
},
307+
308+
########################
309+
# HMAC keys operations #
310+
########################
311+
re.compile('^/projects/(?P<project>[^/]+)/hmacKeys/(?P<accessId>[^/]+)$'): lambda params, data, match: {
312+
'gcs.op': 'hmacKeys.delete',
313+
'gcs.projectId': unquote(match.group('project')),
314+
'gcs.accessId': unquote(match.group('accessId'))
315+
}
316+
}
317+
}

0 commit comments

Comments
 (0)