Skip to content

Commit 0010b72

Browse files
jonsimantovjonsimantov
authored
Clean up flaky tests a bit by adding std::function support to RunWithRetry/RunFlakyBlock. (#534)
By adding std::function overloads to the flaky retry functions (possible now that stlport is removed), we can clean up a lot of code that uses RunFlakyBlock and RunWithRetry. Refactor flaky test handling in Storage to use RunWithRetry instead of RunFlakyBlock where possible - it's much cleaner. Also add some flaky test handling in Auth and Analytics.
1 parent 74d4235 commit 0010b72

File tree

5 files changed

+478
-402
lines changed

5 files changed

+478
-402
lines changed

‎analytics/integration_test/src/integration_test.cc

Lines changed: 21 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -130,23 +130,28 @@ TEST_F(FirebaseAnalyticsTest, TestLogEventWithMultipleParameters) {
130130
sizeof(kLevelUpParameters) / sizeof(kLevelUpParameters[0]));
131131
}
132132

133-
#if !(TARGET_OS_IPHONE)
134-
// Test is flakey on iPhone due to a known issue in iOS. See b/143656277.
135133
TEST_F(FirebaseAnalyticsTest, TestResettingGivesNewInstanceId) {
136-
firebase::Future<std::string> future =
137-
firebase::analytics::GetAnalyticsInstanceId();
138-
WaitForCompletion(future, "GetAnalyticsInstanceId");
139-
EXPECT_FALSE(future.result()->empty());
140-
std::string instance_id = *future.result();
141-
142-
firebase::analytics::ResetAnalyticsData();
143-
144-
future = firebase::analytics::GetAnalyticsInstanceId();
145-
WaitForCompletion(future, "GetAnalyticsInstanceId after ResetAnalyticsData");
146-
std::string new_instance_id = *future.result();
147-
EXPECT_FALSE(future.result()->empty());
148-
EXPECT_NE(instance_id, new_instance_id);
134+
// Test is flaky on iPhone due to a known issue in iOS. See b/143656277.
135+
if (!RunFlakyBlock([&]() {
136+
firebase::Future<std::string> future =
137+
firebase::analytics::GetAnalyticsInstanceId();
138+
FLAKY_WAIT_FOR_COMPLETION(future, "GetAnalyticsInstanceId");
139+
FLAKY_EXPECT_FALSE(future.result()->empty());
140+
std::string instance_id = *future.result();
141+
142+
firebase::analytics::ResetAnalyticsData();
143+
144+
future = firebase::analytics::GetAnalyticsInstanceId();
145+
FLAKY_WAIT_FOR_COMPLETION(
146+
future, "GetAnalyticsInstanceId after ResetAnalyticsData");
147+
std::string new_instance_id = *future.result();
148+
FLAKY_EXPECT_FALSE(future.result()->empty());
149+
FLAKY_EXPECT_NE(instance_id, new_instance_id);
150+
151+
FLAKY_SUCCESS();
152+
})) {
153+
FAIL() << "Failed, see error log for details.";
154+
}
149155
}
150-
#endif // !(TARGET_OS_IPHONE)
151156

152157
} // namespace firebase_testapp_automated

‎auth/integration_test/src/integration_test.cc

Lines changed: 124 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -719,62 +719,75 @@ TEST_F(FirebaseAuthTest, TestWithCustomEmailAndPassword) {
719719
EXPECT_NE(auth_->current_user(), nullptr);
720720
}
721721

722-
#if !defined(__linux__)
723-
// Test is disabled on linux due to the need to unlock the keystore.
724722
TEST_F(FirebaseAuthTest, TestAuthPersistenceWithAnonymousSignin) {
725-
WaitForCompletion(auth_->SignInAnonymously(), "SignInAnonymously");
726-
ASSERT_NE(auth_->current_user(), nullptr);
727-
EXPECT_TRUE(auth_->current_user()->is_anonymous());
728-
Terminate();
729-
ProcessEvents(2000);
730-
Initialize();
731-
EXPECT_NE(auth_, nullptr);
732-
ASSERT_NE(auth_->current_user(), nullptr);
733-
EXPECT_TRUE(auth_->current_user()->is_anonymous());
734-
DeleteUser();
723+
// Test is disabled on linux due to the need to unlock the keystore.
724+
SKIP_TEST_ON_LINUX;
725+
if (!RunFlakyBlock([&]() {
726+
FLAKY_WAIT_FOR_COMPLETION(auth_->SignInAnonymously(),
727+
"SignInAnonymously");
728+
FLAKY_EXPECT_NONNULL(auth_->current_user());
729+
FLAKY_EXPECT_TRUE(auth_->current_user()->is_anonymous());
730+
Terminate();
731+
ProcessEvents(2000);
732+
Initialize();
733+
FLAKY_EXPECT_NONNULL(auth_);
734+
FLAKY_EXPECT_NONNULL(auth_->current_user());
735+
FLAKY_EXPECT_TRUE(auth_->current_user()->is_anonymous());
736+
DeleteUser();
737+
738+
FLAKY_SUCCESS();
739+
})) {
740+
FAIL() << "Test failed, see log for details.";
741+
}
735742
}
736-
#endif // ! defined(__linux__)
737-
738-
#if !defined(__linux__)
739-
// Test is disabled on linux due to the need to unlock the keychain.
740743
TEST_F(FirebaseAuthTest, TestAuthPersistenceWithEmailSignin) {
741-
std::string email = GenerateEmailAddress();
742-
WaitForCompletion(
743-
auth_->CreateUserWithEmailAndPassword(email.c_str(), kTestPassword),
744-
"CreateUserWithEmailAndPassword");
745-
ASSERT_NE(auth_->current_user(), nullptr);
746-
EXPECT_FALSE(auth_->current_user()->is_anonymous());
747-
std::string prev_provider_id = auth_->current_user()->provider_id();
748-
// Save the old provider ID list so we can make sure it's the same once it's
749-
// loaded again.
750-
std::vector<std::string> prev_provider_data_ids;
751-
for (int i = 0; i < auth_->current_user()->provider_data().size(); i++) {
752-
prev_provider_data_ids.push_back(
753-
auth_->current_user()->provider_data()[i]->provider_id());
754-
}
755-
Terminate();
756-
ProcessEvents(2000);
757-
Initialize();
758-
EXPECT_NE(auth_, nullptr);
759-
ASSERT_NE(auth_->current_user(), nullptr);
760-
EXPECT_FALSE(auth_->current_user()->is_anonymous());
761-
// Make sure the provider IDs are the same as they were before.
762-
EXPECT_EQ(auth_->current_user()->provider_id(), prev_provider_id);
763-
std::vector<std::string> loaded_provider_data_ids;
764-
for (int i = 0; i < auth_->current_user()->provider_data().size(); i++) {
765-
loaded_provider_data_ids.push_back(
766-
auth_->current_user()->provider_data()[i]->provider_id());
744+
// Test is disabled on linux due to the need to unlock the keystore.
745+
SKIP_TEST_ON_LINUX;
746+
747+
if (!RunFlakyBlock([&]() {
748+
std::string email = GenerateEmailAddress();
749+
FLAKY_WAIT_FOR_COMPLETION(
750+
auth_->CreateUserWithEmailAndPassword(email.c_str(), kTestPassword),
751+
"CreateUserWithEmailAndPassword");
752+
FLAKY_EXPECT_NONNULL(auth_->current_user());
753+
FLAKY_EXPECT_FALSE(auth_->current_user()->is_anonymous());
754+
std::string prev_provider_id = auth_->current_user()->provider_id();
755+
// Save the old provider ID list so we can make sure it's the same once
756+
// it's loaded again.
757+
std::vector<std::string> prev_provider_data_ids;
758+
for (int i = 0; i < auth_->current_user()->provider_data().size();
759+
i++) {
760+
prev_provider_data_ids.push_back(
761+
auth_->current_user()->provider_data()[i]->provider_id());
762+
}
763+
Terminate();
764+
ProcessEvents(2000);
765+
Initialize();
766+
FLAKY_EXPECT_NONNULL(auth_);
767+
FLAKY_EXPECT_NONNULL(auth_->current_user());
768+
FLAKY_EXPECT_FALSE(auth_->current_user()->is_anonymous());
769+
// Make sure the provider IDs are the same as they were before.
770+
FLAKY_EXPECT_EQ(auth_->current_user()->provider_id(), prev_provider_id);
771+
std::vector<std::string> loaded_provider_data_ids;
772+
for (int i = 0; i < auth_->current_user()->provider_data().size();
773+
i++) {
774+
loaded_provider_data_ids.push_back(
775+
auth_->current_user()->provider_data()[i]->provider_id());
776+
}
777+
FLAKY_EXPECT_TRUE(loaded_provider_data_ids == prev_provider_data_ids);
778+
779+
// Cleanup, ensure we are signed in as the user so we can delete it.
780+
FLAKY_WAIT_FOR_COMPLETION(
781+
auth_->SignInWithEmailAndPassword(email.c_str(), kTestPassword),
782+
"SignInWithEmailAndPassword");
783+
FLAKY_EXPECT_NONNULL(auth_->current_user());
784+
DeleteUser();
785+
786+
FLAKY_SUCCESS();
787+
})) {
788+
FAIL() << "Test failed, see log for details.";
767789
}
768-
EXPECT_EQ(loaded_provider_data_ids, prev_provider_data_ids);
769-
770-
// Cleanup, ensure we are signed in as the user so we can delete it.
771-
WaitForCompletion(
772-
auth_->SignInWithEmailAndPassword(email.c_str(), kTestPassword),
773-
"SignInWithEmailAndPassword");
774-
EXPECT_NE(auth_->current_user(), nullptr);
775-
DeleteUser();
776790
}
777-
#endif // ! defined(__linux__)
778791

779792
class PhoneListener : public firebase::auth::PhoneAuthProvider::Listener {
780793
public:
@@ -860,57 +873,67 @@ TEST_F(FirebaseAuthTest, TestPhoneAuth) {
860873
// Note: This test requires interactivity on iOS, as it displays a CAPTCHA.
861874
TEST_REQUIRES_USER_INTERACTION;
862875
#endif // TARGET_OS_IPHONE
863-
{
864-
firebase::auth::PhoneAuthProvider& phone_provider =
865-
firebase::auth::PhoneAuthProvider::GetInstance(auth_);
866-
LogDebug("Creating listener.");
867-
PhoneListener listener;
868-
LogDebug("Calling VerifyPhoneNumber.");
869-
// Randomly choose one of the phone numbers to avoid collisions.
870-
const int random_phone_number =
871-
app_framework::GetCurrentTimeInMicroseconds() %
872-
kPhoneAuthTestNumPhoneNumbers;
873-
phone_provider.VerifyPhoneNumber(
874-
kPhoneAuthTestPhoneNumbers[random_phone_number], kPhoneAuthTimeoutMs,
875-
nullptr, &listener);
876-
// Wait for OnCodeSent() callback.
877-
int wait_ms = 0;
878-
LogDebug("Waiting for code send.");
879-
while (listener.waiting_to_send_code()) {
880-
if (wait_ms > kPhoneAuthCodeSendWaitMs) break;
881-
ProcessEvents(kWaitIntervalMs);
882-
wait_ms += kWaitIntervalMs;
883-
}
884-
EXPECT_EQ(listener.on_verification_failed_count(), 0);
885-
LogDebug("Waiting for verification ID.");
886-
// Wait for the listener to have a verification ID.
887-
wait_ms = 0;
888-
while (listener.waiting_for_verification_id()) {
889-
if (wait_ms > kPhoneAuthCompletionWaitMs) break;
890-
ProcessEvents(kWaitIntervalMs);
891-
wait_ms += kWaitIntervalMs;
892-
}
893-
if (listener.on_verification_complete_count() > 0) {
894-
LogDebug("Signing in with automatic verification code.");
895-
WaitForCompletion(auth_->SignInWithCredential(listener.credential()),
896-
"SignInWithCredential(PhoneCredential) automatic");
897-
} else if (listener.on_verification_failed_count() > 0) {
898-
FAIL() << "Automatic verification failed.";
899-
} else {
900-
// Did not automatically verify, submit verification code manually.
901-
EXPECT_GT(listener.on_code_auto_retrieval_time_out_count(), 0);
902-
EXPECT_NE(listener.verification_id(), "");
903-
LogDebug("Signing in with verification code.");
904-
const firebase::auth::Credential phone_credential =
905-
phone_provider.GetCredential(listener.verification_id().c_str(),
906-
kPhoneAuthTestVerificationCode);
907-
908-
WaitForCompletion(auth_->SignInWithCredential(phone_credential),
909-
"SignInWithCredential(PhoneCredential)");
910-
}
876+
if (!RunFlakyBlock([&]() {
877+
{
878+
firebase::auth::PhoneAuthProvider& phone_provider =
879+
firebase::auth::PhoneAuthProvider::GetInstance(auth_);
880+
LogDebug("Creating listener.");
881+
PhoneListener listener;
882+
LogDebug("Calling VerifyPhoneNumber.");
883+
// Randomly choose one of the phone numbers to avoid collisions.
884+
const int random_phone_number =
885+
app_framework::GetCurrentTimeInMicroseconds() %
886+
kPhoneAuthTestNumPhoneNumbers;
887+
phone_provider.VerifyPhoneNumber(
888+
kPhoneAuthTestPhoneNumbers[random_phone_number],
889+
kPhoneAuthTimeoutMs, nullptr, &listener);
890+
// Wait for OnCodeSent() callback.
891+
int wait_ms = 0;
892+
LogDebug("Waiting for code send.");
893+
while (listener.waiting_to_send_code()) {
894+
if (wait_ms > kPhoneAuthCodeSendWaitMs) break;
895+
ProcessEvents(kWaitIntervalMs);
896+
wait_ms += kWaitIntervalMs;
897+
}
898+
FLAKY_EXPECT_EQ(listener.on_verification_failed_count(), 0);
899+
LogDebug("Waiting for verification ID.");
900+
// Wait for the listener to have a verification ID.
901+
wait_ms = 0;
902+
while (listener.waiting_for_verification_id()) {
903+
if (wait_ms > kPhoneAuthCompletionWaitMs) break;
904+
ProcessEvents(kWaitIntervalMs);
905+
wait_ms += kWaitIntervalMs;
906+
}
907+
if (listener.on_verification_complete_count() > 0) {
908+
LogDebug("Signing in with automatic verification code.");
909+
FLAKY_WAIT_FOR_COMPLETION(
910+
auth_->SignInWithCredential(listener.credential()),
911+
"SignInWithCredential(PhoneCredential) automatic");
912+
} else if (listener.on_verification_failed_count() > 0) {
913+
LogError("Automatic verification failed.");
914+
FLAKY_FAIL();
915+
} else {
916+
// Did not automatically verify, submit verification code manually.
917+
FLAKY_EXPECT_TRUE(listener.on_code_auto_retrieval_time_out_count() >
918+
0);
919+
FLAKY_EXPECT_NE(listener.verification_id(), "");
920+
LogDebug("Signing in with verification code.");
921+
const firebase::auth::Credential phone_credential =
922+
phone_provider.GetCredential(listener.verification_id().c_str(),
923+
kPhoneAuthTestVerificationCode);
924+
925+
FLAKY_WAIT_FOR_COMPLETION(
926+
auth_->SignInWithCredential(phone_credential),
927+
"SignInWithCredential(PhoneCredential)");
928+
}
929+
}
930+
ProcessEvents(1000);
931+
DeleteUser();
932+
933+
FLAKY_SUCCESS();
934+
})) {
935+
FAIL() << "Phone auth failed, see log for details.";
911936
}
912-
ProcessEvents(1000);
913-
DeleteUser();
914937
}
915938

916939
#if defined(ENABLE_OAUTH_TESTS)

‎messaging/integration_test/src/integration_test.cc

Lines changed: 31 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -579,32 +579,39 @@ TEST_F(FirebaseMessagingTest, TestChangingListener) {
579579
EXPECT_TRUE(RequestPermission());
580580
EXPECT_TRUE(WaitForToken());
581581

582-
// Back up the previous listener object and create a new one.
583-
firebase::messaging::PollableListener* old_listener_ = shared_listener_;
584-
// WaitForMessage() uses whatever shared_listener_ is set to.
585-
shared_listener_ = new firebase::messaging::PollableListener();
586-
firebase::messaging::SetListener(shared_listener_);
587-
// Pause a moment to make sure old listeners are deleted.
588-
ProcessEvents(1000);
582+
if (!RunFlakyBlock([&]() {
583+
// Back up the previous listener object and create a new one.
584+
firebase::messaging::PollableListener* old_listener_ = shared_listener_;
585+
// WaitForMessage() uses whatever shared_listener_ is set to.
586+
shared_listener_ = new firebase::messaging::PollableListener();
587+
firebase::messaging::SetListener(shared_listener_);
588+
// Pause a moment to make sure old listeners are deleted.
589+
ProcessEvents(1000);
590+
591+
std::string unique_id = GetUniqueMessageId();
592+
const char kNotificationTitle[] = "New Listener Test";
593+
const char kNotificationBody[] = "New Listener Test notification body";
594+
SendTestMessage(
595+
shared_token_->c_str(), kNotificationTitle, kNotificationBody,
596+
{{"message", "Hello, world!"}, {"unique_id", unique_id}});
597+
LogDebug("Waiting for message.");
598+
firebase::messaging::Message message;
599+
FLAKY_EXPECT_TRUE(WaitForMessage(&message));
600+
FLAKY_EXPECT_EQ(message.data["unique_id"], unique_id);
601+
if (message.notification) {
602+
FLAKY_EXPECT_EQ(message.notification->title, kNotificationTitle);
603+
FLAKY_EXPECT_EQ(message.notification->body, kNotificationBody);
604+
}
589605

590-
std::string unique_id = GetUniqueMessageId();
591-
const char kNotificationTitle[] = "New Listener Test";
592-
const char kNotificationBody[] = "New Listener Test notification body";
593-
SendTestMessage(shared_token_->c_str(), kNotificationTitle, kNotificationBody,
594-
{{"message", "Hello, world!"}, {"unique_id", unique_id}});
595-
LogDebug("Waiting for message.");
596-
firebase::messaging::Message message;
597-
EXPECT_TRUE(WaitForMessage(&message));
598-
EXPECT_EQ(message.data["unique_id"], unique_id);
599-
if (message.notification) {
600-
EXPECT_EQ(message.notification->title, kNotificationTitle);
601-
EXPECT_EQ(message.notification->body, kNotificationBody);
602-
}
606+
// Set back to the previous listener.
607+
firebase::messaging::SetListener(old_listener_);
608+
delete shared_listener_;
609+
shared_listener_ = old_listener_;
603610

604-
// Set back to the previous listener.
605-
firebase::messaging::SetListener(old_listener_);
606-
delete shared_listener_;
607-
shared_listener_ = old_listener_;
611+
FLAKY_SUCCESS();
612+
})) {
613+
FAIL() << "Test failed, see error log for details.";
614+
}
608615
}
609616

610617
TEST_F(FirebaseMessagingTest, DeliverMetricsToBigQuery) {

0 commit comments

Comments
 (0)