|
1 | | -import base64 |
2 | | -import boto3 |
3 | | -import glob |
4 | | -import hashlib |
5 | 1 | import os |
6 | | -import subprocess |
7 | | -import sys |
8 | | -import tempfile |
9 | 2 |
|
10 | | -from sentry_sdk.consts import VERSION as SDK_VERSION |
11 | | -from sentry_sdk.utils import get_git_revision |
12 | | - |
13 | | -AWS_REGION_NAME = "us-east-1" |
14 | 3 | AWS_CREDENTIALS = { |
15 | 4 | "aws_access_key_id": os.environ["SENTRY_PYTHON_TEST_AWS_ACCESS_KEY_ID"], |
16 | 5 | "aws_secret_access_key": os.environ["SENTRY_PYTHON_TEST_AWS_SECRET_ACCESS_KEY"], |
17 | 6 | } |
18 | | -AWS_LAMBDA_EXECUTION_ROLE_NAME = "lambda-ex" |
19 | | -AWS_LAMBDA_EXECUTION_ROLE_ARN = None |
20 | | - |
21 | | - |
22 | | -def _install_dependencies(base_dir, subprocess_kwargs): |
23 | | - """ |
24 | | - Installs dependencies for AWS Lambda function |
25 | | - """ |
26 | | - setup_cfg = os.path.join(base_dir, "setup.cfg") |
27 | | - with open(setup_cfg, "w") as f: |
28 | | - f.write("[install]\nprefix=") |
29 | | - |
30 | | - # Install requirements for Lambda Layer (these are more limited than the SDK requirements, |
31 | | - # because Lambda does not support the newest versions of some packages) |
32 | | - subprocess.check_call( |
33 | | - [ |
34 | | - sys.executable, |
35 | | - "-m", |
36 | | - "pip", |
37 | | - "install", |
38 | | - "-r", |
39 | | - "requirements-aws-lambda-layer.txt", |
40 | | - "--target", |
41 | | - base_dir, |
42 | | - ], |
43 | | - **subprocess_kwargs, |
44 | | - ) |
45 | | - # Install requirements used for testing |
46 | | - subprocess.check_call( |
47 | | - [ |
48 | | - sys.executable, |
49 | | - "-m", |
50 | | - "pip", |
51 | | - "install", |
52 | | - "mock==3.0.0", |
53 | | - "funcsigs", |
54 | | - "--target", |
55 | | - base_dir, |
56 | | - ], |
57 | | - **subprocess_kwargs, |
58 | | - ) |
59 | | - # Create a source distribution of the Sentry SDK (in parent directory of base_dir) |
60 | | - subprocess.check_call( |
61 | | - [ |
62 | | - sys.executable, |
63 | | - "setup.py", |
64 | | - "sdist", |
65 | | - "--dist-dir", |
66 | | - os.path.dirname(base_dir), |
67 | | - ], |
68 | | - **subprocess_kwargs, |
69 | | - ) |
70 | | - # Install the created Sentry SDK source distribution into the target directory |
71 | | - # Do not install the dependencies of the SDK, because they where installed by requirements-aws-lambda-layer.txt above |
72 | | - source_distribution_archive = glob.glob( |
73 | | - "{}/*.tar.gz".format(os.path.dirname(base_dir)) |
74 | | - )[0] |
75 | | - subprocess.check_call( |
76 | | - [ |
77 | | - sys.executable, |
78 | | - "-m", |
79 | | - "pip", |
80 | | - "install", |
81 | | - source_distribution_archive, |
82 | | - "--no-deps", |
83 | | - "--target", |
84 | | - base_dir, |
85 | | - ], |
86 | | - **subprocess_kwargs, |
87 | | - ) |
88 | | - |
89 | | - |
90 | | -def _create_lambda_function_zip(base_dir): |
91 | | - """ |
92 | | - Zips the given base_dir omitting Python cache files |
93 | | - """ |
94 | | - subprocess.run( |
95 | | - [ |
96 | | - "zip", |
97 | | - "-q", |
98 | | - "-x", |
99 | | - "**/__pycache__/*", |
100 | | - "-r", |
101 | | - "lambda-function-package.zip", |
102 | | - "./", |
103 | | - ], |
104 | | - cwd=base_dir, |
105 | | - check=True, |
106 | | - ) |
107 | | - |
108 | | - |
109 | | -def _create_lambda_package( |
110 | | - base_dir, code, initial_handler, layer, syntax_check, subprocess_kwargs |
111 | | -): |
112 | | - """ |
113 | | - Creates deployable packages (as zip files) for AWS Lambda function |
114 | | - and optional the accompanying Sentry Lambda layer |
115 | | - """ |
116 | | - if initial_handler: |
117 | | - # If Initial handler value is provided i.e. it is not the default |
118 | | - # `test_lambda.test_handler`, then create another dir level so that our path is |
119 | | - # test_dir.test_lambda.test_handler |
120 | | - test_dir_path = os.path.join(base_dir, "test_dir") |
121 | | - python_init_file = os.path.join(test_dir_path, "__init__.py") |
122 | | - os.makedirs(test_dir_path) |
123 | | - with open(python_init_file, "w"): |
124 | | - # Create __init__ file to make it a python package |
125 | | - pass |
126 | | - |
127 | | - test_lambda_py = os.path.join(base_dir, "test_dir", "test_lambda.py") |
128 | | - else: |
129 | | - test_lambda_py = os.path.join(base_dir, "test_lambda.py") |
130 | | - |
131 | | - with open(test_lambda_py, "w") as f: |
132 | | - f.write(code) |
133 | | - |
134 | | - if syntax_check: |
135 | | - # Check file for valid syntax first, and that the integration does not |
136 | | - # crash when not running in Lambda (but rather a local deployment tool |
137 | | - # such as chalice's) |
138 | | - subprocess.check_call([sys.executable, test_lambda_py]) |
139 | | - |
140 | | - if layer is None: |
141 | | - _install_dependencies(base_dir, subprocess_kwargs) |
142 | | - _create_lambda_function_zip(base_dir) |
143 | | - |
144 | | - else: |
145 | | - _create_lambda_function_zip(base_dir) |
146 | | - |
147 | | - # Create Lambda layer zip package |
148 | | - from scripts.build_aws_lambda_layer import build_packaged_zip |
149 | | - |
150 | | - build_packaged_zip( |
151 | | - base_dir=base_dir, |
152 | | - make_dist=True, |
153 | | - out_zip_filename="lambda-layer-package.zip", |
154 | | - ) |
155 | | - |
156 | | - |
157 | | -def _get_or_create_lambda_execution_role(): |
158 | | - global AWS_LAMBDA_EXECUTION_ROLE_ARN |
159 | | - |
160 | | - policy = """{ |
161 | | - "Version": "2012-10-17", |
162 | | - "Statement": [ |
163 | | - { |
164 | | - "Effect": "Allow", |
165 | | - "Principal": { |
166 | | - "Service": "lambda.amazonaws.com" |
167 | | - }, |
168 | | - "Action": "sts:AssumeRole" |
169 | | - } |
170 | | - ] |
171 | | - } |
172 | | - """ |
173 | | - iam_client = boto3.client( |
174 | | - "iam", |
175 | | - region_name=AWS_REGION_NAME, |
176 | | - **AWS_CREDENTIALS, |
177 | | - ) |
178 | | - |
179 | | - try: |
180 | | - response = iam_client.get_role(RoleName=AWS_LAMBDA_EXECUTION_ROLE_NAME) |
181 | | - AWS_LAMBDA_EXECUTION_ROLE_ARN = response["Role"]["Arn"] |
182 | | - except iam_client.exceptions.NoSuchEntityException: |
183 | | - # create role for lambda execution |
184 | | - response = iam_client.create_role( |
185 | | - RoleName=AWS_LAMBDA_EXECUTION_ROLE_NAME, |
186 | | - AssumeRolePolicyDocument=policy, |
187 | | - ) |
188 | | - AWS_LAMBDA_EXECUTION_ROLE_ARN = response["Role"]["Arn"] |
189 | | - |
190 | | - # attach policy to role |
191 | | - iam_client.attach_role_policy( |
192 | | - RoleName=AWS_LAMBDA_EXECUTION_ROLE_NAME, |
193 | | - PolicyArn="arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", |
194 | | - ) |
195 | | - |
196 | | - |
197 | | -def get_boto_client(): |
198 | | - _get_or_create_lambda_execution_role() |
199 | | - |
200 | | - return boto3.client( |
201 | | - "lambda", |
202 | | - region_name=AWS_REGION_NAME, |
203 | | - **AWS_CREDENTIALS, |
204 | | - ) |
205 | | - |
206 | | - |
207 | | -def run_lambda_function( |
208 | | - client, |
209 | | - runtime, |
210 | | - code, |
211 | | - payload, |
212 | | - add_finalizer, |
213 | | - syntax_check=True, |
214 | | - timeout=30, |
215 | | - layer=None, |
216 | | - initial_handler=None, |
217 | | - subprocess_kwargs=(), |
218 | | -): |
219 | | - """ |
220 | | - Creates a Lambda function with the given code, and invokes it. |
221 | | -
|
222 | | - If the same code is run multiple times the function will NOT be |
223 | | - created anew each time but the existing function will be reused. |
224 | | - """ |
225 | | - subprocess_kwargs = dict(subprocess_kwargs) |
226 | | - |
227 | | - # Making a unique function name depending on all the code that is run in it (function code plus SDK version) |
228 | | - # The name needs to be short so the generated event/envelope json blobs are small enough to be output |
229 | | - # in the log result of the Lambda function. |
230 | | - rev = get_git_revision() or SDK_VERSION |
231 | | - function_hash = hashlib.shake_256((code + rev).encode("utf-8")).hexdigest(6) |
232 | | - fn_name = "test_{}".format(function_hash) |
233 | | - full_fn_name = "{}_{}".format( |
234 | | - fn_name, runtime.replace(".", "").replace("python", "py") |
235 | | - ) |
236 | | - |
237 | | - function_exists_in_aws = True |
238 | | - try: |
239 | | - client.get_function( |
240 | | - FunctionName=full_fn_name, |
241 | | - ) |
242 | | - print( |
243 | | - "Lambda function in AWS already existing, taking it (and do not create a local one)" |
244 | | - ) |
245 | | - except client.exceptions.ResourceNotFoundException: |
246 | | - function_exists_in_aws = False |
247 | 7 |
|
248 | | - if not function_exists_in_aws: |
249 | | - tmp_base_dir = tempfile.gettempdir() |
250 | | - base_dir = os.path.join(tmp_base_dir, fn_name) |
251 | | - dir_already_existing = os.path.isdir(base_dir) |
252 | 8 |
|
253 | | - if dir_already_existing: |
254 | | - print("Local Lambda function directory already exists, skipping creation") |
255 | | - |
256 | | - if not dir_already_existing: |
257 | | - os.mkdir(base_dir) |
258 | | - _create_lambda_package( |
259 | | - base_dir, code, initial_handler, layer, syntax_check, subprocess_kwargs |
260 | | - ) |
261 | | - |
262 | | - @add_finalizer |
263 | | - def clean_up(): |
264 | | - # this closes the web socket so we don't get a |
265 | | - # ResourceWarning: unclosed <ssl.SSLSocket ... > |
266 | | - # warning on every test |
267 | | - # based on https://github.com/boto/botocore/pull/1810 |
268 | | - # (if that's ever merged, this can just become client.close()) |
269 | | - session = client._endpoint.http_session |
270 | | - managers = [session._manager] + list(session._proxy_managers.values()) |
271 | | - for manager in managers: |
272 | | - manager.clear() |
273 | | - |
274 | | - layers = [] |
275 | | - environment = {} |
276 | | - handler = initial_handler or "test_lambda.test_handler" |
277 | | - |
278 | | - if layer is not None: |
279 | | - with open( |
280 | | - os.path.join(base_dir, "lambda-layer-package.zip"), "rb" |
281 | | - ) as lambda_layer_zip: |
282 | | - response = client.publish_layer_version( |
283 | | - LayerName="python-serverless-sdk-test", |
284 | | - Description="Created as part of testsuite for getsentry/sentry-python", |
285 | | - Content={"ZipFile": lambda_layer_zip.read()}, |
286 | | - ) |
287 | | - |
288 | | - layers = [response["LayerVersionArn"]] |
289 | | - handler = ( |
290 | | - "sentry_sdk.integrations.init_serverless_sdk.sentry_lambda_handler" |
291 | | - ) |
292 | | - environment = { |
293 | | - "Variables": { |
294 | | - "SENTRY_INITIAL_HANDLER": initial_handler |
295 | | - or "test_lambda.test_handler", |
296 | | - "SENTRY_DSN": "https://[email protected]/123", |
297 | | - "SENTRY_TRACES_SAMPLE_RATE": "1.0", |
298 | | - } |
299 | | - } |
300 | | - |
301 | | - try: |
302 | | - with open( |
303 | | - os.path.join(base_dir, "lambda-function-package.zip"), "rb" |
304 | | - ) as lambda_function_zip: |
305 | | - client.create_function( |
306 | | - Description="Created as part of testsuite for getsentry/sentry-python", |
307 | | - FunctionName=full_fn_name, |
308 | | - Runtime=runtime, |
309 | | - Timeout=timeout, |
310 | | - Role=AWS_LAMBDA_EXECUTION_ROLE_ARN, |
311 | | - Handler=handler, |
312 | | - Code={"ZipFile": lambda_function_zip.read()}, |
313 | | - Environment=environment, |
314 | | - Layers=layers, |
315 | | - ) |
316 | | - |
317 | | - waiter = client.get_waiter("function_active_v2") |
318 | | - waiter.wait(FunctionName=full_fn_name) |
319 | | - except client.exceptions.ResourceConflictException: |
320 | | - print( |
321 | | - "Lambda function already exists, this is fine, we will just invoke it." |
322 | | - ) |
323 | | - |
324 | | - response = client.invoke( |
325 | | - FunctionName=full_fn_name, |
326 | | - InvocationType="RequestResponse", |
327 | | - LogType="Tail", |
328 | | - Payload=payload, |
329 | | - ) |
330 | 9 |
|
331 | | - assert 200 <= response["StatusCode"] < 300, response |
332 | | - return response |
333 | 10 |
|
334 | 11 |
|
335 | 12 | # This is for inspecting new Python runtime environments in AWS Lambda |
|
0 commit comments