Skip to content

Commit 3e2fa2f

Browse files
igorborgestigorborgest
authored
Merge pull request #233 from awslabs/dev
Dev
2 parents e5e645d + 022f5ac commit 3e2fa2f

File tree

10 files changed

+127
-56
lines changed

10 files changed

+127
-56
lines changed

‎awswrangler/s3.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -271,7 +271,7 @@ def delete_objects(
271271
else:
272272
cpus: int = _utils.ensure_cpu_count(use_threads=use_threads)
273273
with concurrent.futures.ThreadPoolExecutor(max_workers=cpus) as executor:
274-
executor.map(_delete_objects, repeat(bucket), chunks, repeat(client_s3))
274+
list(executor.map(_delete_objects, repeat(bucket), chunks, repeat(client_s3)))
275275

276276

277277
def _split_paths_by_bucket(paths: List[str]) -> Dict[str, List[str]]:

‎testing/deploy-cloudformation.sh‎ renamed to ‎testing/cloudformation.sh‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,20 @@
11
#!/usr/bin/env bash
2-
set -ex
2+
set -e
33

44
cfn-lint -t cloudformation.yaml
55
rm -rf temp.yaml
66
cfn-flip -c -l -n cloudformation.yaml temp.yaml
77
cfn-lint -t temp.yaml
88
mv temp.yaml cloudformation.yaml
99

10+
read -rp "Databases password (e.g. 123456Ab): " password
11+
1012
aws cloudformation deploy \
1113
--template-file cloudformation.yaml \
12-
--stack-name aws-data-wrangler-test \
14+
--stack-name aws-data-wrangler \
1315
--capabilities CAPABILITY_IAM \
14-
--parameter-overrides $(cat parameters.properties)
16+
--parameter-overrides DatabasesPassword="$password"
1517

1618
aws cloudformation update-termination-protection \
1719
--enable-termination-protection \
18-
--stack-name aws-data-wrangler-test
20+
--stack-name aws-data-wrangler

‎testing/cloudformation.yaml‎

Lines changed: 111 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,97 @@
11
AWSTemplateFormatVersion: 2010-09-09
22
Description: |
3-
AWS Data Wrangler Test Infrastructure
3+
AWS Data Wrangler Development Infrastructure
44
Parameters:
5-
VpcId:
6-
Type: String
7-
Description: Redshift VPC ID
8-
SubnetId:
9-
Type: String
10-
Description: Redshift Subnet ID
11-
SubnetId2:
12-
Type: String
13-
Description: Redshift Subnet ID
14-
SubnetAz:
15-
Type: String
16-
Description: Subnet AZ
175
DatabasesPassword:
186
Type: String
197
Description: Password for all databases
208
NoEcho: true
21-
AWSUserForTests:
22-
Type: String
23-
Description: AWS User that will running the tests on the CLI
249
Resources:
10+
VPC:
11+
Type: AWS::EC2::VPC
12+
Properties:
13+
CidrBlock: 10.19.224.0/19
14+
EnableDnsSupport: true
15+
EnableDnsHostnames: true
16+
Tags:
17+
- Key: Name
18+
Value: aws-data-wrangler
19+
InternetGateway:
20+
Type: AWS::EC2::InternetGateway
21+
Properties:
22+
Tags:
23+
- Key: Name
24+
Value: aws-data-wrangler
25+
InternetGatewayAttachment:
26+
Type: AWS::EC2::VPCGatewayAttachment
27+
Properties:
28+
InternetGatewayId:
29+
Ref: InternetGateway
30+
VpcId:
31+
Ref: VPC
32+
PublicSubnet1:
33+
Type: AWS::EC2::Subnet
34+
Properties:
35+
VpcId:
36+
Ref: VPC
37+
AvailabilityZone:
38+
Fn::Select:
39+
- 0
40+
- Fn::GetAZs: ''
41+
CidrBlock: 10.19.229.0/24
42+
MapPublicIpOnLaunch: true
43+
Tags:
44+
- Key: Name
45+
Value: aws-data-wrangler Public Subnet (AZ1)
46+
PublicSubnet2:
47+
Type: AWS::EC2::Subnet
48+
Properties:
49+
VpcId:
50+
Ref: VPC
51+
AvailabilityZone:
52+
Fn::Select:
53+
- 1
54+
- Fn::GetAZs: ''
55+
CidrBlock: 10.19.230.0/24
56+
MapPublicIpOnLaunch: true
57+
Tags:
58+
- Key: Name
59+
Value: aws-data-wrangler Public Subnet (AZ2)
60+
PublicRouteTable:
61+
Type: AWS::EC2::RouteTable
62+
Properties:
63+
VpcId:
64+
Ref: VPC
65+
Tags:
66+
- Key: Name
67+
Value: aws-data-wrangler Public Routes
68+
DefaultPublicRoute:
69+
Type: AWS::EC2::Route
70+
DependsOn: InternetGatewayAttachment
71+
Properties:
72+
RouteTableId:
73+
Ref: PublicRouteTable
74+
DestinationCidrBlock: 0.0.0.0/0
75+
GatewayId:
76+
Ref: InternetGateway
77+
PublicSubnet1RouteTableAssociation:
78+
Type: AWS::EC2::SubnetRouteTableAssociation
79+
Properties:
80+
RouteTableId:
81+
Ref: PublicRouteTable
82+
SubnetId:
83+
Ref: PublicSubnet1
84+
PublicSubnet2RouteTableAssociation:
85+
Type: AWS::EC2::SubnetRouteTableAssociation
86+
Properties:
87+
RouteTableId:
88+
Ref: PublicRouteTable
89+
SubnetId:
90+
Ref: PublicSubnet2
2591
KmsKeyAlias:
2692
Type: AWS::KMS::Alias
2793
Properties:
28-
AliasName: alias/aws-data-wrangler-test-key
94+
AliasName: alias/aws-data-wrangler-key
2995
TargetKeyId:
3096
Ref: KmsKey
3197
KmsKey:
@@ -34,7 +100,7 @@ Resources:
34100
Description: Aws Data Wrangler Test Key.
35101
KeyPolicy:
36102
Version: '2012-10-17'
37-
Id: aws-data-wrangler-test-key
103+
Id: aws-data-wrangler-key
38104
Statement:
39105
- Sid: Enable IAM User Permissions
40106
Effect: Allow
@@ -46,8 +112,7 @@ Resources:
46112
- Sid: Allow administration of the key
47113
Effect: Allow
48114
Principal:
49-
AWS:
50-
Fn::Sub: arn:aws:iam::${AWS::AccountId}:user/${AWSUserForTests}
115+
AWS: '*'
51116
Action:
52117
- kms:Create*
53118
- kms:Describe*
@@ -142,12 +207,12 @@ Resources:
142207
Properties:
143208
Description: AWS Data Wrangler Test Arena - Redshift Subnet Group
144209
SubnetIds:
145-
- Ref: SubnetId
210+
- Ref: PublicSubnet1
146211
DatabaseSecurityGroup:
147212
Type: AWS::EC2::SecurityGroup
148213
Properties:
149214
VpcId:
150-
Ref: VpcId
215+
Ref: VPC
151216
GroupDescription: AWS Data Wrangler Test Arena - Redshift security group
152217
DatabaseSecurityGroupIngress:
153218
Type: AWS::EC2::SecurityGroupIngress
@@ -185,7 +250,7 @@ Resources:
185250
CatalogId:
186251
Ref: AWS::AccountId
187252
DatabaseInput:
188-
Name: awswrangler_test
253+
Name: aws_data_wrangler
189254
Description: AWS Data Wrangler Test Arena - Glue Database
190255
LogGroup:
191256
Type: AWS::Logs::LogGroup
@@ -201,8 +266,8 @@ Resources:
201266
Properties:
202267
DBSubnetGroupDescription: RDS Database Subnet Group
203268
SubnetIds:
204-
- Ref: SubnetId
205-
- Ref: SubnetId2
269+
- Ref: PublicSubnet1
270+
- Ref: PublicSubnet2
206271
AuroraRole:
207272
Type: AWS::IAM::Role
208273
Properties:
@@ -232,14 +297,15 @@ Resources:
232297
PostgresqlParameterGroup:
233298
Type: AWS::RDS::DBClusterParameterGroup
234299
Properties:
235-
Description: Postgres 10
236-
Family: aurora-postgresql10
300+
Description: Postgres 11
301+
Family: aurora-postgresql11
237302
Parameters:
238303
apg_plan_mgmt.capture_plan_baselines: 'off'
239304
AuroraClusterPostgresql:
240305
Type: AWS::RDS::DBCluster
241306
Properties:
242307
Engine: aurora-postgresql
308+
EngineVersion: '11.6'
243309
DBClusterIdentifier: postgresql-cluster-wrangler
244310
MasterUsername: test
245311
MasterUserPassword:
@@ -261,6 +327,7 @@ Resources:
261327
Type: AWS::RDS::DBInstance
262328
Properties:
263329
Engine: aurora-postgresql
330+
EngineVersion: '11.6'
264331
DBInstanceIdentifier: postgresql-instance-wrangler
265332
DBClusterIdentifier:
266333
Ref: AuroraClusterPostgresql
@@ -290,6 +357,7 @@ Resources:
290357
Type: AWS::RDS::DBCluster
291358
Properties:
292359
Engine: aurora-mysql
360+
EngineVersion: '5.7'
293361
DBClusterIdentifier: mysql-cluster-wrangler
294362
MasterUsername: test
295363
MasterUserPassword:
@@ -311,6 +379,7 @@ Resources:
311379
Type: AWS::RDS::DBInstance
312380
Properties:
313381
Engine: aurora-mysql
382+
EngineVersion: '5.7'
314383
DBInstanceIdentifier: mysql-instance-wrangler
315384
DBClusterIdentifier:
316385
Ref: AuroraClusterMysql
@@ -328,11 +397,13 @@ Resources:
328397
ConnectionType: JDBC
329398
PhysicalConnectionRequirements:
330399
AvailabilityZone:
331-
Ref: SubnetAz
400+
Fn::Select:
401+
- 0
402+
- Fn::GetAZs: ''
332403
SecurityGroupIdList:
333404
- Ref: DatabaseSecurityGroup
334405
SubnetId:
335-
Ref: SubnetId
406+
Ref: PublicSubnet1
336407
ConnectionProperties:
337408
JDBC_CONNECTION_URL:
338409
Fn::Sub: jdbc:redshift://${Redshift.Endpoint.Address}:${Redshift.Endpoint.Port}/test
@@ -350,11 +421,13 @@ Resources:
350421
ConnectionType: JDBC
351422
PhysicalConnectionRequirements:
352423
AvailabilityZone:
353-
Ref: SubnetAz
424+
Fn::Select:
425+
- 0
426+
- Fn::GetAZs: ''
354427
SecurityGroupIdList:
355428
- Ref: DatabaseSecurityGroup
356429
SubnetId:
357-
Ref: SubnetId
430+
Ref: PublicSubnet1
358431
ConnectionProperties:
359432
JDBC_CONNECTION_URL:
360433
Fn::Sub: jdbc:postgresql://${AuroraInstancePostgresql.Endpoint.Address}:${AuroraInstancePostgresql.Endpoint.Port}/postgres
@@ -372,11 +445,13 @@ Resources:
372445
ConnectionType: JDBC
373446
PhysicalConnectionRequirements:
374447
AvailabilityZone:
375-
Ref: SubnetAz
448+
Fn::Select:
449+
- 0
450+
- Fn::GetAZs: ''
376451
SecurityGroupIdList:
377452
- Ref: DatabaseSecurityGroup
378453
SubnetId:
379-
Ref: SubnetId
454+
Ref: PublicSubnet1
380455
ConnectionProperties:
381456
JDBC_CONNECTION_URL:
382457
Fn::Sub: jdbc:mysql://${AuroraInstanceMysql.Endpoint.Address}:${AuroraInstanceMysql.Endpoint.Port}/test
@@ -398,7 +473,7 @@ Resources:
398473
- AttributeName: attr_range
399474
KeyType: RANGE
400475
BillingMode: PAY_PER_REQUEST
401-
TableName: aws-data-wrangler-test
476+
TableName: aws-data-wrangler
402477
Outputs:
403478
BucketName:
404479
Value:
@@ -449,11 +524,11 @@ Outputs:
449524
Description: LogStream name.
450525
SubnetId:
451526
Value:
452-
Ref: SubnetId
527+
Ref: PublicSubnet1
453528
Description: Subnet ID
454529
SubnetId2:
455530
Value:
456-
Ref: SubnetId2
531+
Ref: PublicSubnet2
457532
Description: Subnet ID 2
458533
PostgresqlAddress:
459534
Value:

‎testing/parameters.properties‎

Lines changed: 0 additions & 6 deletions
This file was deleted.

‎testing/test_awswrangler/test_cloudwatch.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414

1515
@pytest.fixture(scope="module")
1616
def cloudformation_outputs():
17-
response = boto3.client("cloudformation").describe_stacks(StackName="aws-data-wrangler-test")
17+
response = boto3.client("cloudformation").describe_stacks(StackName="aws-data-wrangler")
1818
outputs = {}
1919
for output in response.get("Stacks")[0].get("Outputs"):
2020
outputs[output.get("OutputKey")] = output.get("OutputValue")

‎testing/test_awswrangler/test_data_lake.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222

2323
@pytest.fixture(scope="module")
2424
def cloudformation_outputs():
25-
response = boto3.client("cloudformation").describe_stacks(StackName="aws-data-wrangler-test")
25+
response = boto3.client("cloudformation").describe_stacks(StackName="aws-data-wrangler")
2626
outputs = {}
2727
for output in response.get("Stacks")[0].get("Outputs"):
2828
outputs[output.get("OutputKey")] = output.get("OutputValue")
@@ -66,7 +66,7 @@ def external_schema(cloudformation_outputs, database):
6666

6767
@pytest.fixture(scope="module")
6868
def workgroup0(bucket):
69-
wkg_name = "awswrangler_test_0"
69+
wkg_name = "aws_data_wrangler_0"
7070
client = boto3.client("athena")
7171
wkgs = client.list_work_groups()
7272
wkgs = [x["Name"] for x in wkgs["WorkGroups"]]
@@ -87,7 +87,7 @@ def workgroup0(bucket):
8787

8888
@pytest.fixture(scope="module")
8989
def workgroup1(bucket):
90-
wkg_name = "awswrangler_test_1"
90+
wkg_name = "aws_data_wrangler_1"
9191
client = boto3.client("athena")
9292
wkgs = client.list_work_groups()
9393
wkgs = [x["Name"] for x in wkgs["WorkGroups"]]
@@ -111,7 +111,7 @@ def workgroup1(bucket):
111111

112112
@pytest.fixture(scope="module")
113113
def workgroup2(bucket, kms_key):
114-
wkg_name = "awswrangler_test_2"
114+
wkg_name = "aws_data_wrangler_2"
115115
client = boto3.client("athena")
116116
wkgs = client.list_work_groups()
117117
wkgs = [x["Name"] for x in wkgs["WorkGroups"]]
@@ -135,7 +135,7 @@ def workgroup2(bucket, kms_key):
135135

136136
@pytest.fixture(scope="module")
137137
def workgroup3(bucket, kms_key):
138-
wkg_name = "awswrangler_test_3"
138+
wkg_name = "aws_data_wrangler_3"
139139
client = boto3.client("athena")
140140
wkgs = client.list_work_groups()
141141
wkgs = [x["Name"] for x in wkgs["WorkGroups"]]

‎testing/test_awswrangler/test_db.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818

1919
@pytest.fixture(scope="module")
2020
def cloudformation_outputs():
21-
response = boto3.client("cloudformation").describe_stacks(StackName="aws-data-wrangler-test")
21+
response = boto3.client("cloudformation").describe_stacks(StackName="aws-data-wrangler")
2222
outputs = {}
2323
for output in response.get("Stacks")[0].get("Outputs"):
2424
outputs[output.get("OutputKey")] = output.get("OutputValue")

‎testing/test_awswrangler/test_emr.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313

1414
@pytest.fixture(scope="module")
1515
def cloudformation_outputs():
16-
response = boto3.client("cloudformation").describe_stacks(StackName="aws-data-wrangler-test")
16+
response = boto3.client("cloudformation").describe_stacks(StackName="aws-data-wrangler")
1717
outputs = {}
1818
for output in response.get("Stacks")[0].get("Outputs"):
1919
outputs[output.get("OutputKey")] = output.get("OutputValue")

0 commit comments

Comments
 (0)