Skip to content

Commit 7b82cb9

Browse files
lrafeeiTimPansinohmstepanekumaannamalai
authored
Fix botocore tests for botocore v1.28.1+ (#675)
* Fix botocore tests for botocore v1.28.1+ Co-authored-by: Timothy Pansino <[email protected]> * Fix boto3 tests for botocore v1.28.1+ Co-authored-by: Timothy Pansino <[email protected]> Co-authored-by: Hannah Stepanek <[email protected]> Co-authored-by: Uma Annamalai <[email protected]> * Fix boto3 tests for python 2.7 Co-authored-by: Timothy Pansino <[email protected]> Co-authored-by: Hannah Stepanek <[email protected]> Co-authored-by: Uma Annamalai <[email protected]> Co-authored-by: Timothy Pansino <[email protected]> Co-authored-by: Hannah Stepanek <[email protected]> Co-authored-by: Uma Annamalai <[email protected]> Co-authored-by: Timothy Pansino <[email protected]>
1 parent a66a33a commit 7b82cb9

File tree

3 files changed

+123
-118
lines changed

3 files changed

+123
-118
lines changed

‎tests/external_boto3/test_boto3_s3.py‎

Lines changed: 61 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -18,106 +18,106 @@
1818
import boto3
1919
import botocore
2020
import moto
21+
from testing_support.fixtures import (
22+
override_application_settings,
23+
validate_transaction_metrics,
24+
)
25+
from testing_support.validators.validate_span_events import validate_span_events
2126

2227
from newrelic.api.background_task import background_task
23-
from testing_support.fixtures import (validate_transaction_metrics,
24-
override_application_settings)
25-
from testing_support.validators.validate_span_events import (
26-
validate_span_events)
2728

28-
MOTO_VERSION = tuple(int(v) for v in moto.__version__.split('.')[:3])
29+
MOTO_VERSION = tuple(int(v) for v in moto.__version__.split(".")[:3])
2930

3031
# patch earlier versions of moto to support py37
3132
if sys.version_info >= (3, 7) and MOTO_VERSION <= (1, 3, 1):
3233
import re
34+
3335
moto.packages.responses.responses.re._pattern_type = re.Pattern
3436

35-
AWS_ACCESS_KEY_ID = 'AAAAAAAAAAAACCESSKEY'
36-
AWS_SECRET_ACCESS_KEY = 'AAAAAASECRETKEY'
37-
AWS_REGION_NAME = 'us-west-2'
37+
AWS_ACCESS_KEY_ID = "AAAAAAAAAAAACCESSKEY"
38+
AWS_SECRET_ACCESS_KEY = "AAAAAASECRETKEY" # nosec
39+
AWS_REGION_NAME = "us-west-2"
40+
41+
TEST_BUCKET = "python-agent-test-%s" % uuid.uuid4()
42+
43+
BOTOCORE_VERSION = tuple(map(int, botocore.__version__.split(".")))
3844

39-
TEST_BUCKET = 'python-agent-test-%s' % uuid.uuid4()
4045

41-
BOTOCORE_VERSION = tuple(map(int, botocore.__version__.split('.')))
4246
if BOTOCORE_VERSION < (1, 7, 41):
43-
S3_URL = 's3-us-west-2.amazonaws.com'
47+
S3_URL = "s3-us-west-2.amazonaws.com"
48+
EXPECTED_BUCKET_URL = "https://%s/%s" % (S3_URL, TEST_BUCKET)
49+
EXPECTED_KEY_URL = EXPECTED_BUCKET_URL + "/hello_world"
50+
elif BOTOCORE_VERSION < (1, 28):
51+
S3_URL = "s3.us-west-2.amazonaws.com"
52+
EXPECTED_BUCKET_URL = "https://%s/%s" % (S3_URL, TEST_BUCKET)
53+
EXPECTED_KEY_URL = EXPECTED_BUCKET_URL + "/hello_world"
4454
else:
45-
S3_URL = 's3.us-west-2.amazonaws.com'
55+
S3_URL = "%s.s3.us-west-2.amazonaws.com" % TEST_BUCKET
56+
EXPECTED_BUCKET_URL = "https://%s/" % S3_URL
57+
EXPECTED_KEY_URL = EXPECTED_BUCKET_URL + "hello_world"
4658

47-
expected_http_url = 'https://%s/%s' % (S3_URL, TEST_BUCKET)
4859

4960
_s3_scoped_metrics = [
50-
('External/%s/botocore/GET' % S3_URL, 2),
51-
('External/%s/botocore/PUT' % S3_URL, 2),
52-
('External/%s/botocore/DELETE' % S3_URL, 2),
61+
("External/%s/botocore/GET" % S3_URL, 2),
62+
("External/%s/botocore/PUT" % S3_URL, 2),
63+
("External/%s/botocore/DELETE" % S3_URL, 2),
5364
]
5465

5566
_s3_rollup_metrics = [
56-
('External/all', 6),
57-
('External/allOther', 6),
58-
('External/%s/all' % S3_URL, 6),
59-
('External/%s/botocore/GET' % S3_URL, 2),
60-
('External/%s/botocore/PUT' % S3_URL, 2),
61-
('External/%s/botocore/DELETE' % S3_URL, 2),
67+
("External/all", 6),
68+
("External/allOther", 6),
69+
("External/%s/all" % S3_URL, 6),
70+
("External/%s/botocore/GET" % S3_URL, 2),
71+
("External/%s/botocore/PUT" % S3_URL, 2),
72+
("External/%s/botocore/DELETE" % S3_URL, 2),
6273
]
6374

6475

65-
@override_application_settings({'distributed_tracing.enabled': True})
66-
@validate_span_events(exact_agents={'aws.operation': 'CreateBucket'}, count=1)
67-
@validate_span_events(exact_agents={'aws.operation': 'PutObject'}, count=1)
68-
@validate_span_events(exact_agents={'aws.operation': 'ListObjects'}, count=1)
69-
@validate_span_events(exact_agents={'aws.operation': 'GetObject'}, count=1)
70-
@validate_span_events(exact_agents={'aws.operation': 'DeleteObject'}, count=1)
71-
@validate_span_events(exact_agents={'aws.operation': 'DeleteBucket'}, count=1)
72-
@validate_span_events(
73-
exact_agents={'http.url': expected_http_url}, count=3)
74-
@validate_span_events(
75-
exact_agents={'http.url': expected_http_url + '/hello_world'}, count=3)
76+
@override_application_settings({"distributed_tracing.enabled": True})
77+
@validate_span_events(exact_agents={"aws.operation": "CreateBucket"}, count=1)
78+
@validate_span_events(exact_agents={"aws.operation": "PutObject"}, count=1)
79+
@validate_span_events(exact_agents={"aws.operation": "ListObjects"}, count=1)
80+
@validate_span_events(exact_agents={"aws.operation": "GetObject"}, count=1)
81+
@validate_span_events(exact_agents={"aws.operation": "DeleteObject"}, count=1)
82+
@validate_span_events(exact_agents={"aws.operation": "DeleteBucket"}, count=1)
83+
@validate_span_events(exact_agents={"http.url": EXPECTED_BUCKET_URL}, count=3)
84+
@validate_span_events(exact_agents={"http.url": EXPECTED_KEY_URL}, count=3)
7685
@validate_transaction_metrics(
77-
'test_boto3_s3:test_s3',
78-
scoped_metrics=_s3_scoped_metrics,
79-
rollup_metrics=_s3_rollup_metrics,
80-
background_task=True)
86+
"test_boto3_s3:test_s3", scoped_metrics=_s3_scoped_metrics, rollup_metrics=_s3_rollup_metrics, background_task=True
87+
)
8188
@background_task()
8289
@moto.mock_s3
8390
def test_s3():
8491
client = boto3.client(
85-
's3',
86-
aws_access_key_id=AWS_ACCESS_KEY_ID,
87-
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
88-
region_name=AWS_REGION_NAME,
92+
"s3",
93+
aws_access_key_id=AWS_ACCESS_KEY_ID,
94+
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
95+
region_name=AWS_REGION_NAME,
8996
)
9097

9198
# Create bucket
92-
resp = client.create_bucket(
93-
Bucket=TEST_BUCKET,
94-
CreateBucketConfiguration={'LocationConstraint': AWS_REGION_NAME}
95-
)
96-
assert resp['ResponseMetadata']['HTTPStatusCode'] == 200
99+
resp = client.create_bucket(Bucket=TEST_BUCKET, CreateBucketConfiguration={"LocationConstraint": AWS_REGION_NAME})
100+
assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200
97101

98102
# Put object
99-
resp = client.put_object(
100-
Bucket=TEST_BUCKET,
101-
Key='hello_world',
102-
Body=b'hello_world_content'
103-
)
104-
assert resp['ResponseMetadata']['HTTPStatusCode'] == 200
103+
resp = client.put_object(Bucket=TEST_BUCKET, Key="hello_world", Body=b"hello_world_content")
104+
assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200
105105

106106
# List bucket
107107
resp = client.list_objects(Bucket=TEST_BUCKET)
108-
assert resp['ResponseMetadata']['HTTPStatusCode'] == 200
109-
assert len(resp['Contents']) == 1
110-
assert resp['Contents'][0]['Key'] == 'hello_world'
108+
assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200
109+
assert len(resp["Contents"]) == 1
110+
assert resp["Contents"][0]["Key"] == "hello_world"
111111

112112
# Get object
113-
resp = client.get_object(Bucket=TEST_BUCKET, Key='hello_world')
114-
assert resp['ResponseMetadata']['HTTPStatusCode'] == 200
115-
assert resp['Body'].read() == b'hello_world_content'
113+
resp = client.get_object(Bucket=TEST_BUCKET, Key="hello_world")
114+
assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200
115+
assert resp["Body"].read() == b"hello_world_content"
116116

117117
# Delete object
118-
resp = client.delete_object(Bucket=TEST_BUCKET, Key='hello_world')
119-
assert resp['ResponseMetadata']['HTTPStatusCode'] == 204
118+
resp = client.delete_object(Bucket=TEST_BUCKET, Key="hello_world")
119+
assert resp["ResponseMetadata"]["HTTPStatusCode"] == 204
120120

121121
# Delete bucket
122122
resp = client.delete_bucket(Bucket=TEST_BUCKET)
123-
assert resp['ResponseMetadata']['HTTPStatusCode'] == 204
123+
assert resp["ResponseMetadata"]["HTTPStatusCode"] == 204

‎tests/external_botocore/test_botocore_s3.py‎

Lines changed: 58 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -15,101 +15,104 @@
1515
import sys
1616
import uuid
1717

18+
import botocore
1819
import botocore.session
1920
import moto
21+
from testing_support.fixtures import (
22+
override_application_settings,
23+
validate_transaction_metrics,
24+
)
25+
from testing_support.validators.validate_span_events import validate_span_events
2026

2127
from newrelic.api.background_task import background_task
22-
from testing_support.fixtures import (validate_transaction_metrics,
23-
override_application_settings)
24-
from testing_support.validators.validate_span_events import (
25-
validate_span_events)
2628

27-
MOTO_VERSION = tuple(int(v) for v in moto.__version__.split('.')[:3])
29+
MOTO_VERSION = tuple(int(v) for v in moto.__version__.split(".")[:3])
30+
BOTOCORE_VERSION = tuple(int(v) for v in botocore.__version__.split(".")[:3])
31+
2832

2933
# patch earlier versions of moto to support py37
3034
if sys.version_info >= (3, 7) and MOTO_VERSION <= (1, 3, 1):
3135
import re
36+
3237
moto.packages.responses.responses.re._pattern_type = re.Pattern
3338

34-
AWS_ACCESS_KEY_ID = 'AAAAAAAAAAAACCESSKEY'
35-
AWS_SECRET_ACCESS_KEY = 'AAAAAASECRETKEY'
36-
AWS_REGION = 'us-east-1'
39+
AWS_ACCESS_KEY_ID = "AAAAAAAAAAAACCESSKEY"
40+
AWS_SECRET_ACCESS_KEY = "AAAAAASECRETKEY" # nosec
41+
AWS_REGION = "us-east-1"
3742

38-
TEST_BUCKET = 'python-agent-test-%s' % uuid.uuid4()
39-
S3_URL = 's3.amazonaws.com'
40-
expected_http_url = 'https://%s/%s' % (S3_URL, TEST_BUCKET)
43+
TEST_BUCKET = "python-agent-test-%s" % uuid.uuid4()
44+
if BOTOCORE_VERSION >= (1, 28):
45+
S3_URL = "%s.s3.amazonaws.com" % TEST_BUCKET
46+
EXPECTED_BUCKET_URL = "https://%s/" % S3_URL
47+
EXPECTED_KEY_URL = EXPECTED_BUCKET_URL + "hello_world"
48+
else:
49+
S3_URL = "s3.amazonaws.com"
50+
EXPECTED_BUCKET_URL = "https://%s/%s" % (S3_URL, TEST_BUCKET)
51+
EXPECTED_KEY_URL = EXPECTED_BUCKET_URL + "/hello_world"
4152

4253

4354
_s3_scoped_metrics = [
44-
('External/s3.amazonaws.com/botocore/GET', 2),
45-
('External/s3.amazonaws.com/botocore/PUT', 2),
46-
('External/s3.amazonaws.com/botocore/DELETE', 2),
55+
("External/%s/botocore/GET" % S3_URL, 2),
56+
("External/%s/botocore/PUT" % S3_URL, 2),
57+
("External/%s/botocore/DELETE" % S3_URL, 2),
4758
]
4859

4960
_s3_rollup_metrics = [
50-
('External/all', 6),
51-
('External/allOther', 6),
52-
('External/s3.amazonaws.com/all', 6),
53-
('External/s3.amazonaws.com/botocore/GET', 2),
54-
('External/s3.amazonaws.com/botocore/PUT', 2),
55-
('External/s3.amazonaws.com/botocore/DELETE', 2),
61+
("External/all", 6),
62+
("External/allOther", 6),
63+
("External/%s/all" % S3_URL, 6),
64+
("External/%s/botocore/GET" % S3_URL, 2),
65+
("External/%s/botocore/PUT" % S3_URL, 2),
66+
("External/%s/botocore/DELETE" % S3_URL, 2),
5667
]
5768

5869

59-
@override_application_settings({'distributed_tracing.enabled': True})
60-
@validate_span_events(exact_agents={'aws.operation': 'CreateBucket'}, count=1)
61-
@validate_span_events(exact_agents={'aws.operation': 'PutObject'}, count=1)
62-
@validate_span_events(exact_agents={'aws.operation': 'ListObjects'}, count=1)
63-
@validate_span_events(exact_agents={'aws.operation': 'GetObject'}, count=1)
64-
@validate_span_events(exact_agents={'aws.operation': 'DeleteObject'}, count=1)
65-
@validate_span_events(exact_agents={'aws.operation': 'DeleteBucket'}, count=1)
66-
@validate_span_events(
67-
exact_agents={'http.url': expected_http_url}, count=3)
68-
@validate_span_events(
69-
exact_agents={'http.url': expected_http_url + '/hello_world'}, count=3)
70+
@override_application_settings({"distributed_tracing.enabled": True})
71+
@validate_span_events(exact_agents={"aws.operation": "CreateBucket"}, count=1)
72+
@validate_span_events(exact_agents={"aws.operation": "PutObject"}, count=1)
73+
@validate_span_events(exact_agents={"aws.operation": "ListObjects"}, count=1)
74+
@validate_span_events(exact_agents={"aws.operation": "GetObject"}, count=1)
75+
@validate_span_events(exact_agents={"aws.operation": "DeleteObject"}, count=1)
76+
@validate_span_events(exact_agents={"aws.operation": "DeleteBucket"}, count=1)
77+
@validate_span_events(exact_agents={"http.url": EXPECTED_BUCKET_URL}, count=3)
78+
@validate_span_events(exact_agents={"http.url": EXPECTED_KEY_URL}, count=3)
7079
@validate_transaction_metrics(
71-
'test_botocore_s3:test_s3',
72-
scoped_metrics=_s3_scoped_metrics,
73-
rollup_metrics=_s3_rollup_metrics,
74-
background_task=True)
80+
"test_botocore_s3:test_s3",
81+
scoped_metrics=_s3_scoped_metrics,
82+
rollup_metrics=_s3_rollup_metrics,
83+
background_task=True,
84+
)
7585
@background_task()
7686
@moto.mock_s3
7787
def test_s3():
7888
session = botocore.session.get_session()
7989
client = session.create_client(
80-
's3',
81-
region_name=AWS_REGION,
82-
aws_access_key_id=AWS_ACCESS_KEY_ID,
83-
aws_secret_access_key=AWS_SECRET_ACCESS_KEY
90+
"s3", region_name=AWS_REGION, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY
8491
)
8592

8693
# Create bucket
8794
resp = client.create_bucket(Bucket=TEST_BUCKET)
88-
assert resp['ResponseMetadata']['HTTPStatusCode'] == 200
95+
assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200
8996

9097
# Put object
91-
resp = client.put_object(
92-
Bucket=TEST_BUCKET,
93-
Key='hello_world',
94-
Body=b'hello_world_content'
95-
)
96-
assert resp['ResponseMetadata']['HTTPStatusCode'] == 200
98+
resp = client.put_object(Bucket=TEST_BUCKET, Key="hello_world", Body=b"hello_world_content")
99+
assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200
97100

98101
# List bucket
99102
resp = client.list_objects(Bucket=TEST_BUCKET)
100-
assert resp['ResponseMetadata']['HTTPStatusCode'] == 200
101-
assert len(resp['Contents']) == 1
102-
assert resp['Contents'][0]['Key'] == 'hello_world'
103+
assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200
104+
assert len(resp["Contents"]) == 1
105+
assert resp["Contents"][0]["Key"] == "hello_world"
103106

104107
# Get object
105-
resp = client.get_object(Bucket=TEST_BUCKET, Key='hello_world')
106-
assert resp['ResponseMetadata']['HTTPStatusCode'] == 200
107-
assert resp['Body'].read() == b'hello_world_content'
108+
resp = client.get_object(Bucket=TEST_BUCKET, Key="hello_world")
109+
assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200
110+
assert resp["Body"].read() == b"hello_world_content"
108111

109112
# Delete object
110-
resp = client.delete_object(Bucket=TEST_BUCKET, Key='hello_world')
111-
assert resp['ResponseMetadata']['HTTPStatusCode'] == 204
113+
resp = client.delete_object(Bucket=TEST_BUCKET, Key="hello_world")
114+
assert resp["ResponseMetadata"]["HTTPStatusCode"] == 204
112115

113116
# Delete bucket
114117
resp = client.delete_bucket(Bucket=TEST_BUCKET)
115-
assert resp['ResponseMetadata']['HTTPStatusCode'] == 204
118+
assert resp["ResponseMetadata"]["HTTPStatusCode"] == 204

‎tox.ini‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,8 @@ envlist =
9595
python-datastore_sqlite-{py27,py37,py38,py39,py310,py311,pypy,pypy37},
9696
memcached-datastore_umemcache-{py27,pypy},
9797
python-external_boto3-{py27,py37,py38,py39,py310,py311}-boto01,
98-
python-external_botocore-{py27,py37,py38,py39,py310,py311},
98+
python-external_botocore-{py27,py37,py38,py39,py310,py311}-botocorelatest,
99+
python-external_botocore-py310-botocore0125,
99100
python-external_feedparser-py27-feedparser{05,06},
100101
python-external_http-{py27,py37,py38,py39,py310,py311,pypy},
101102
python-external_httplib-{py27,py37,py38,py39,py310,py311,pypy,pypy37},
@@ -259,7 +260,8 @@ deps =
259260
external_boto3-boto01: boto3<2.0
260261
external_boto3-boto01: moto<2.0
261262
external_boto3-py27: rsa<4.7.1
262-
external_botocore: botocore
263+
external_botocore-botocorelatest: botocore
264+
external_botocore-botocore0125: botocore<1.26
263265
external_botocore-{py37,py38,py39,py310,py311}: moto[awslambda,ec2,iam]<3.0
264266
external_botocore-py27: rsa<4.7.1
265267
external_botocore-py27: moto[awslambda,ec2,iam]<2.0

0 commit comments

Comments
 (0)