Skip to content

Commit a6e41e6

Browse files
thisisabhashthisisabhash
authored
chore: Add github action for stress tests in all categories (#2648)
* chore: Add github action for stress tests in all categories * Update scheme name * Update amplify configuration file name to a common file name * Update amplify configuration file name for datastore and api * Fix typo and update datastore syncMaxRecords to 500 * Update code * Fix datastore stress tests * Address review comments * Remove unused code
1 parent 233b179 commit a6e41e6

File tree

14 files changed

+271
-36
lines changed

14 files changed

+271
-36
lines changed

‎.github/workflows/stress_test.yml

Lines changed: 109 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,16 +41,121 @@ jobs:
4141
- name: Make directory
4242
run: mkdir -p ~/.aws-amplify/amplify-ios/testconfiguration/
4343

44-
- name: Copy integration test resouces
44+
- name: Copy stress test resouces
4545
uses: ./.github/composite_actions/download_test_configuration
4646
with:
47-
resource_subfolder: auth
47+
resource_subfolder: stresstest
4848
aws_role_to_assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
4949
aws_region: ${{ secrets.AWS_REGION }}
5050
aws_s3_bucket: ${{ secrets.AWS_S3_BUCKET_INTEG_V2 }}
5151

52-
- name: Run Integration test
52+
- name: Run stress test
5353
uses: ./.github/composite_actions/run_xcodebuild_test
5454
with:
5555
project_path: ./AmplifyPlugins/Auth/Tests/AuthHostApp/
56-
scheme: AuthStressTests
56+
scheme: AuthStressTests
57+
58+
geo-stress-test:
59+
needs: prepare-for-test
60+
runs-on: macos-12
61+
environment: IntegrationTest
62+
steps:
63+
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
64+
with:
65+
persist-credentials: false
66+
67+
- name: Make directory
68+
run: mkdir -p ~/.aws-amplify/amplify-ios/testconfiguration/
69+
70+
- name: Copy stress test resouces
71+
uses: ./.github/composite_actions/download_test_configuration
72+
with:
73+
resource_subfolder: stresstest
74+
aws_role_to_assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
75+
aws_region: ${{ secrets.AWS_REGION }}
76+
aws_s3_bucket: ${{ secrets.AWS_S3_BUCKET_INTEG_V2 }}
77+
78+
- name: Run stress test
79+
uses: ./.github/composite_actions/run_xcodebuild_test
80+
with:
81+
project_path: ./AmplifyPlugins/Geo/Tests/GeoHostApp/
82+
scheme: GeoStressTests
83+
84+
85+
storage-stress-test:
86+
needs: prepare-for-test
87+
runs-on: macos-12
88+
environment: IntegrationTest
89+
steps:
90+
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
91+
with:
92+
persist-credentials: false
93+
94+
- name: Make directory
95+
run: mkdir -p ~/.aws-amplify/amplify-ios/testconfiguration/
96+
97+
- name: Copy stress test resouces
98+
uses: ./.github/composite_actions/download_test_configuration
99+
with:
100+
resource_subfolder: stresstest
101+
aws_role_to_assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
102+
aws_region: ${{ secrets.AWS_REGION }}
103+
aws_s3_bucket: ${{ secrets.AWS_S3_BUCKET_INTEG_V2 }}
104+
105+
- name: Run stress test
106+
uses: ./.github/composite_actions/run_xcodebuild_test
107+
with:
108+
project_path: ./AmplifyPlugins/Storage/Tests/StorageHostApp/
109+
scheme: StorageStressTests
110+
111+
datastore-stress-test:
112+
needs: prepare-for-test
113+
runs-on: macos-12
114+
environment: IntegrationTest
115+
steps:
116+
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
117+
with:
118+
persist-credentials: false
119+
120+
- name: Make directory
121+
run: mkdir -p ~/.aws-amplify/amplify-ios/testconfiguration/
122+
123+
- name: Copy stress test resouces
124+
uses: ./.github/composite_actions/download_test_configuration
125+
with:
126+
resource_subfolder: stresstest
127+
aws_role_to_assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
128+
aws_region: ${{ secrets.AWS_REGION }}
129+
aws_s3_bucket: ${{ secrets.AWS_S3_BUCKET_INTEG_V2 }}
130+
131+
- name: Run stress test
132+
uses: ./.github/composite_actions/run_xcodebuild_test
133+
with:
134+
project_path: ./AmplifyPlugins/DataStore/Tests/DataStoreHostApp
135+
scheme: DatastoreStressTests
136+
137+
graphql-api-stress-test:
138+
needs: prepare-for-test
139+
runs-on: macos-12
140+
environment: IntegrationTest
141+
steps:
142+
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
143+
with:
144+
persist-credentials: false
145+
146+
- name: Make directory
147+
run: mkdir -p ~/.aws-amplify/amplify-ios/testconfiguration/
148+
149+
- name: Copy stress test resouces
150+
uses: ./.github/composite_actions/download_test_configuration
151+
with:
152+
resource_subfolder: stresstest
153+
aws_role_to_assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
154+
aws_region: ${{ secrets.AWS_REGION }}
155+
aws_s3_bucket: ${{ secrets.AWS_S3_BUCKET_INTEG_V2 }}
156+
157+
- name: Run stress test
158+
uses: ./.github/composite_actions/run_xcodebuild_test
159+
with:
160+
project_path: ./AmplifyPlugins/API/Tests/APIHostApp
161+
scheme: GraphQLAPIStressTests

‎AmplifyPlugins/Analytics/Tests/AnalyticsHostApp/AnalyticsStressTests/AnalyticsStressTests.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import Network
1515

1616
final class AnalyticsStressTests: XCTestCase {
1717

18-
static let amplifyConfiguration = "testconfiguration/AWSAnalyticsStressTests-amplifyconfiguration"
18+
static let amplifyConfiguration = "testconfiguration/AWSAmplifyStressTests-amplifyconfiguration"
1919
static let analyticsPluginKey = "awsPinpointAnalyticsPlugin"
2020
let concurrencyLimit = 50
2121

‎AmplifyPlugins/Analytics/Tests/AnalyticsHostApp/AnalyticsStressTests/README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,10 @@ Auth category is also required for signing with AWS Pinpoint service and request
1717

1818
3. `amplify push`
1919

20-
4. Copy `amplifyconfiguration.json` as `AWSAnalyticsStressTests-amplifyconfiguration.json` inside `~/.aws-amplify/amplify-ios/testconfiguration/`
20+
4. Copy `amplifyconfiguration.json` as `AWSAmplifyStressTests-amplifyconfiguration.json` inside `~/.aws-amplify/amplify-ios/testconfiguration/`
2121

2222
```perl
23-
cp amplifyconfiguration.json ~/.aws-amplify/amplify-ios/testconfiguration/AWSAnalyticsStressTests-amplifyconfiguration.json
23+
cp amplifyconfiguration.json ~/.aws-amplify/amplify-ios/testconfiguration/AWSAmplifyStressTests-amplifyconfiguration.json
2424
```
2525

2626
5. You can now run all of the integration tests.

‎AmplifyPlugins/Auth/Tests/AuthHostApp/AuthHostApp.xcodeproj/project.pbxproj

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
681DFEAB28E747B80000C36A /* AsyncTesting.swift in Sources */ = {isa = PBXBuildFile; fileRef = 681DFEA828E747B80000C36A /* AsyncTesting.swift */; };
3131
681DFEAC28E747B80000C36A /* AsyncExpectation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 681DFEA928E747B80000C36A /* AsyncExpectation.swift */; };
3232
681DFEAD28E747B80000C36A /* XCTestCase+AsyncTesting.swift in Sources */ = {isa = PBXBuildFile; fileRef = 681DFEAA28E747B80000C36A /* XCTestCase+AsyncTesting.swift */; };
33+
970333F0295D793B0019981E /* AuthStressBaseTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 970333EF295D793B0019981E /* AuthStressBaseTest.swift */; };
3334
9737C74E287E208400DA0D2B /* AuthRememberDeviceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9737C74D287E208400DA0D2B /* AuthRememberDeviceTests.swift */; };
3435
9737C7502880BFD600DA0D2B /* AuthForgetDeviceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9737C74F2880BFD600DA0D2B /* AuthForgetDeviceTests.swift */; };
3536
97829201286B802E000DE190 /* AuthResetPasswordTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 97829200286B802E000DE190 /* AuthResetPasswordTests.swift */; };
@@ -39,7 +40,6 @@
3940
97914B4E2955099A002000EA /* XCTestCase+AsyncTesting.swift in Sources */ = {isa = PBXBuildFile; fileRef = 681DFEAA28E747B80000C36A /* XCTestCase+AsyncTesting.swift */; };
4041
97914B4F295509A2002000EA /* AuthStressTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 97914AD72954FE0A002000EA /* AuthStressTests.swift */; };
4142
97914B51295509AB002000EA /* README.md in Resources */ = {isa = PBXBuildFile; fileRef = 97914AF6295503D4002000EA /* README.md */; };
42-
97914B5529552D0C002000EA /* AWSAuthBaseTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 485CB5AF27B61EAA006CCEC7 /* AWSAuthBaseTest.swift */; };
4343
97914B5629552D2B002000EA /* AuthSessionHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = 485CB5B727B61F0F006CCEC7 /* AuthSessionHelper.swift */; };
4444
97914B5729552D2B002000EA /* AuthSignInHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = 485CB5B827B61F0F006CCEC7 /* AuthSignInHelper.swift */; };
4545
97B370C52878DA5A00F1C088 /* AuthFetchDeviceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 97B370C42878DA5A00F1C088 /* AuthFetchDeviceTests.swift */; };
@@ -93,6 +93,7 @@
9393
681DFEA828E747B80000C36A /* AsyncTesting.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AsyncTesting.swift; sourceTree = "<group>"; };
9494
681DFEA928E747B80000C36A /* AsyncExpectation.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AsyncExpectation.swift; sourceTree = "<group>"; };
9595
681DFEAA28E747B80000C36A /* XCTestCase+AsyncTesting.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "XCTestCase+AsyncTesting.swift"; sourceTree = "<group>"; };
96+
970333EF295D793B0019981E /* AuthStressBaseTest.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AuthStressBaseTest.swift; sourceTree = "<group>"; };
9697
9737C74D287E208400DA0D2B /* AuthRememberDeviceTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AuthRememberDeviceTests.swift; sourceTree = "<group>"; };
9798
9737C74F2880BFD600DA0D2B /* AuthForgetDeviceTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AuthForgetDeviceTests.swift; sourceTree = "<group>"; };
9899
97829200286B802E000DE190 /* AuthResetPasswordTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AuthResetPasswordTests.swift; sourceTree = "<group>"; };
@@ -306,6 +307,7 @@
306307
children = (
307308
97914AF6295503D4002000EA /* README.md */,
308309
97914AD72954FE0A002000EA /* AuthStressTests.swift */,
310+
970333EF295D793B0019981E /* AuthStressBaseTest.swift */,
309311
);
310312
path = AuthStressTests;
311313
sourceTree = "<group>";
@@ -551,8 +553,8 @@
551553
files = (
552554
97914B5629552D2B002000EA /* AuthSessionHelper.swift in Sources */,
553555
97914B5729552D2B002000EA /* AuthSignInHelper.swift in Sources */,
554-
97914B5529552D0C002000EA /* AWSAuthBaseTest.swift in Sources */,
555556
97914B4F295509A2002000EA /* AuthStressTests.swift in Sources */,
557+
970333F0295D793B0019981E /* AuthStressBaseTest.swift in Sources */,
556558
97914B4C2955099A002000EA /* AsyncTesting.swift in Sources */,
557559
97914B4D2955099A002000EA /* AsyncExpectation.swift in Sources */,
558560
97914B4E2955099A002000EA /* XCTestCase+AsyncTesting.swift in Sources */,

‎AmplifyPlugins/Auth/Tests/AuthHostApp/AuthStressTests/AuthStressBaseTest.swift

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
//
2+
// Copyright Amazon.com Inc. or its affiliates.
3+
// All Rights Reserved.
4+
//
5+
// SPDX-License-Identifier: Apache-2.0
6+
//
7+
8+
import XCTest
9+
@testable import Amplify
10+
import AWSCognitoAuthPlugin
11+
12+
class AuthStressBaseTest: XCTestCase {
13+
14+
let networkTimeout = TimeInterval(5)
15+
16+
var defaultTestEmail = "test-\(UUID().uuidString)@amazon.com"
17+
var defaultTestPassword = UUID().uuidString
18+
19+
let amplifyConfigurationFile = "testconfiguration/AWSAmplifyStressTests-amplifyconfiguration"
20+
let credentialsFile = "testconfiguration/AWSAmplifyStressTests-credentials"
21+
22+
var amplifyConfiguration: AmplifyConfiguration!
23+
24+
override func setUp() async throws {
25+
try await super.setUp()
26+
initializeAmplify()
27+
_ = await Amplify.Auth.signOut()
28+
}
29+
30+
override func tearDown() async throws {
31+
try await super.tearDown()
32+
await Amplify.reset()
33+
}
34+
35+
func initializeAmplify() {
36+
do {
37+
let configuration = try TestConfigHelper.retrieveAmplifyConfiguration(
38+
forResource: amplifyConfigurationFile)
39+
amplifyConfiguration = configuration
40+
41+
let credentialsConfiguration = (try? TestConfigHelper.retrieveCredentials(forResource: credentialsFile)) ?? [:]
42+
defaultTestEmail = credentialsConfiguration["test_email_1"] ?? defaultTestEmail
43+
defaultTestPassword = credentialsConfiguration["password"] ?? defaultTestPassword
44+
let authPlugin = AWSCognitoAuthPlugin()
45+
try Amplify.add(plugin: authPlugin)
46+
try Amplify.configure(configuration)
47+
Amplify.Logging.logLevel = .verbose
48+
print("Amplify configured with auth plugin")
49+
} catch {
50+
print(error)
51+
initializeWithLocalResources()
52+
}
53+
}
54+
55+
func initializeWithLocalResources() {
56+
let region = JSONValue(stringLiteral: "xx")
57+
let userPoolID = JSONValue(stringLiteral: "xx")
58+
let userPooldAppClientID = JSONValue(stringLiteral: "xx")
59+
let userPooldAppClientSecret = JSONValue(stringLiteral: "xx")
60+
61+
let identityPoolID = JSONValue(stringLiteral: "xx")
62+
do {
63+
let authConfiguration = AuthCategoryConfiguration(plugins: [
64+
"awsCognitoAuthPlugin": [
65+
"UserAgent": "aws-amplify/cli",
66+
"Version": "0.1.0",
67+
"IdentityManager": [
68+
"Default": []
69+
],
70+
"CredentialsProvider": [
71+
"CognitoIdentity": [
72+
"Default": [
73+
"PoolId": identityPoolID,
74+
"Region": region
75+
]
76+
]
77+
],
78+
"CognitoUserPool": [
79+
"Default": [
80+
"PoolId": userPoolID,
81+
"AppClientId": userPooldAppClientID,
82+
"Region": region
83+
]
84+
]]]
85+
)
86+
let configuration = AmplifyConfiguration(auth: authConfiguration)
87+
let authPlugin = AWSCognitoAuthPlugin()
88+
try Amplify.add(plugin: authPlugin)
89+
try Amplify.configure(configuration)
90+
} catch {
91+
print(error)
92+
XCTFail("Amplify configuration failed")
93+
}
94+
}
95+
}
96+
97+
class TestConfigHelper {
98+
99+
static func retrieveAmplifyConfiguration(forResource: String) throws -> AmplifyConfiguration {
100+
101+
let data = try retrieve(forResource: forResource)
102+
return try AmplifyConfiguration.decodeAmplifyConfiguration(from: data)
103+
}
104+
105+
static func retrieveCredentials(forResource: String) throws -> [String: String] {
106+
let data = try retrieve(forResource: forResource)
107+
108+
let jsonOptional = try JSONSerialization.jsonObject(with: data, options: []) as? [String: String]
109+
guard let json = jsonOptional else {
110+
throw TestConfigError.jsonError("Could not deserialize `\(forResource)` into JSON object")
111+
}
112+
113+
return json
114+
}
115+
116+
private static func retrieve(forResource: String) throws -> Data {
117+
guard let path = Bundle(for: self).path(forResource: forResource, ofType: "json") else {
118+
throw TestConfigError.bundlePathError("Could not retrieve configuration file: \(forResource)")
119+
}
120+
121+
let url = URL(fileURLWithPath: path)
122+
return try Data(contentsOf: url)
123+
}
124+
}
125+
126+
enum TestConfigError: Error {
127+
128+
case jsonError(String)
129+
130+
case bundlePathError(String)
131+
}

‎AmplifyPlugins/Auth/Tests/AuthHostApp/AuthStressTests/AuthStressTests.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import XCTest
1010
import AWSCognitoAuthPlugin
1111
import AWSPluginsCore
1212

13-
final class AuthStressTests: AWSAuthBaseTest {
13+
final class AuthStressTests: AuthStressBaseTest {
1414

1515
let concurrencyLimit = 50
1616

‎AmplifyPlugins/Auth/Tests/AuthHostApp/AuthStressTests/README.md

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,5 @@ Successfully added resource amplifyintegtest locally
8282
amplify push
8383
```
8484

85-
This will create a amplifyconfiguration.json file in your local, copy that file to `~/.aws-amplify/amplify-ios/testconfiguration/` and rename as `AWSAuthStressTests-amplifyconfiguration.json`.
85+
This will create a amplifyconfiguration.json file in your local, copy that file to `~/.aws-amplify/amplify-ios/testconfiguration/` and rename as `AWSAmplifyStressTests-amplifyconfiguration.json`.
8686

87-
For Auth Device tests:
88-
Follow steps here (https://docs.amplify.aws/lib/auth/device_features/q/platform/ios/#configure-auth-category)[https://docs.amplify.aws/lib/auth/device_features/q/platform/ios/#configure-auth-category] and select "Always" for "Do you want to remember your user's devices?"

‎AmplifyPlugins/DataStore/Tests/DataStoreHostApp/DataStoreStressTests/DataStoreStressBaseTest.swift

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,13 @@ import AWSAPIPlugin
1717

1818
class DataStoreStressBaseTest: XCTestCase {
1919

20-
static let amplifyConfigurationFile = "testconfiguration/AWSDataStoreStressTests-amplifyconfiguration"
20+
static let amplifyConfigurationFile = "testconfiguration/AWSAmplifyStressTests-amplifyconfiguration"
2121
let concurrencyLimit = 50
2222
let networkTimeout = TimeInterval(180)
2323

2424
func setUp(withModels models: AmplifyModelRegistration, logLevel: LogLevel = .error) async {
25+
2526
continueAfterFailure = false
26-
await Amplify.reset()
2727
Amplify.Logging.logLevel = logLevel
2828

2929
do {
@@ -37,6 +37,10 @@ class DataStoreStressBaseTest: XCTestCase {
3737
}
3838
}
3939

40+
override func tearDown() async throws {
41+
await Amplify.reset()
42+
}
43+
4044
func stopDataStore() async throws {
4145
try await Amplify.DataStore.stop()
4246
}
@@ -70,6 +74,6 @@ class DataStoreStressBaseTest: XCTestCase {
7074

7175
try await Amplify.DataStore.start()
7276

73-
await waitForExpectations(timeout: 100.0)
77+
await waitForExpectations(timeout: networkTimeout)
7478
}
7579
}

0 commit comments

Comments
 (0)