Skip to content

Commit 1812a0f

Browse files
jilu1jilu1
committed
MQE-1800: Allow generate:tests to continue running even if one test fails generation
1 parent 7ff4c3d commit 1812a0f

File tree

7 files changed

+97
-46
lines changed

7 files changed

+97
-46
lines changed

‎src/Magento/FunctionalTestingFramework/Console/GenerateSuiteCommand.php

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -67,26 +67,35 @@ protected function execute(InputInterface $input, OutputInterface $output)
6767

6868
$suites = $input->getArgument('suites');
6969

70+
$generated = 0;
7071
foreach ($suites as $suite) {
7172
try {
7273
SuiteGenerator::getInstance()->generateSuite($suite);
7374
if ($output->isVerbose()) {
7475
$output->writeLn("suite $suite generated");
7576
}
77+
$generated++;
7678
} catch (FastFailException $e) {
7779
throw $e;
7880
} catch (\Exception $e) {
7981
}
8082
}
8183

8284
if (empty(GenerationErrorHandler::getInstance()->getAllErrors())) {
83-
$output->writeln("Suites Generated");
84-
return 0;
85+
if ($generated > 0) {
86+
$output->writeln("Suites Generated");
87+
return 0;
88+
}
8589
} else {
8690
GenerationErrorHandler::getInstance()->printErrorSummary();
8791
GenerationErrorHandler::getInstance()->reset();
88-
$output->writeln("Suites Generated (with failures)");
89-
return 1;
92+
if ($generated > 0) {
93+
$output->writeln("Suites Generated (with errors)");
94+
return 1;
95+
}
9096
}
97+
98+
$output->writeln("No Suite Generated");
99+
return 1;
91100
}
92101
}

‎src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
169169
} else {
170170
GenerationErrorHandler::getInstance()->printErrorSummary();
171171
GenerationErrorHandler::getInstance()->reset();
172-
$output->writeln("Generate Tests Command Run (with failures)");
172+
$output->writeln("Generate Tests Command Run (with errors)");
173173
return 1;
174174
}
175175
}

‎src/Magento/FunctionalTestingFramework/DataTransport/Auth/WebApiAuth.php

Lines changed: 31 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
namespace Magento\FunctionalTestingFramework\DataTransport\Auth;
88

9+
use Magento\FunctionalTestingFramework\Exceptions\FastFailException;
910
use Magento\FunctionalTestingFramework\Util\MftfGlobals;
1011
use Magento\FunctionalTestingFramework\DataTransport\Protocol\CurlInterface;
1112
use Magento\FunctionalTestingFramework\DataTransport\Protocol\CurlTransport;
@@ -42,7 +43,7 @@ class WebApiAuth
4243
* @param string $username
4344
* @param string $password
4445
* @return string
45-
* @throws TestFrameworkException
46+
* @throws FastFailException
4647
*/
4748
public static function getAdminToken($username = null, $password = null)
4849
{
@@ -56,32 +57,37 @@ public static function getAdminToken($username = null, $password = null)
5657
'MAGENTO_ADMIN_USERNAME' => getenv('MAGENTO_ADMIN_USERNAME'),
5758
'MAGENTO_ADMIN_PASSWORD' => getenv('MAGENTO_ADMIN_PASSWORD'),
5859
];
59-
throw new TestFrameworkException($message, $context);
60+
throw new FastFailException($message, $context);
6061
}
6162

6263
if (isset(self::$adminAuthTokens[$login])) {
6364
return self::$adminAuthTokens[$login];
6465
}
6566

66-
$authUrl = MftfGlobals::getWebApiBaseUrl() . self::PATH_ADMIN_AUTH;
67+
try {
68+
$authUrl = MftfGlobals::getWebApiBaseUrl() . self::PATH_ADMIN_AUTH;
6769

68-
$data = [
69-
'username' => $login,
70-
'password' => $password
71-
];
70+
$data = [
71+
'username' => $login,
72+
'password' => $password
73+
];
7274

73-
if (Tfa::isEnabled()) {
74-
$authUrl = MftfGlobals::getWebApiBaseUrl() . Tfa::getProviderWebApiAuthEndpoint('google');
75-
$data['otp'] = OTP::getOTP();
76-
}
75+
if (Tfa::isEnabled()) {
76+
$authUrl = MftfGlobals::getWebApiBaseUrl() . Tfa::getProviderWebApiAuthEndpoint('google');
77+
$data['otp'] = OTP::getOTP();
78+
}
7779

78-
$transport = new CurlTransport();
79-
$transport->write(
80-
$authUrl,
81-
json_encode($data, JSON_PRETTY_PRINT),
82-
CurlInterface::POST,
83-
self::$headers
84-
);
80+
$transport = new CurlTransport();
81+
$transport->write(
82+
$authUrl,
83+
json_encode($data, JSON_PRETTY_PRINT),
84+
CurlInterface::POST,
85+
self::$headers
86+
);
87+
} catch (TestFrameworkException $e) {
88+
$message = "Cannot retrieve API token with credentials. Please check configurations in .env.\n";
89+
throw new FastFailException($message . $e->getMessage(), $e->getContext());
90+
}
8591

8692
try {
8793
$response = $transport->read();
@@ -97,12 +103,16 @@ public static function getAdminToken($username = null, $password = null)
97103
$errMessage = $e->getMessage();
98104
}
99105

100-
$message = 'Cannot retrieve API token with credentials. Please check the following credentials';
101-
$message .= Tfa::isEnabled() ? ' and 2FA settings:' : ':' . PHP_EOL;
106+
$message = 'Cannot retrieve API token with credentials. Please check the following configurations';
107+
try {
108+
// No exception will ever throw from here
109+
$message .= Tfa::isEnabled() ? ' and 2FA settings:' : ':' . PHP_EOL;
110+
} catch (TestFrameworkException $e) {
111+
}
102112
$message .= "username: {$login}" . PHP_EOL;
103113
$message .= "password: {$password}" . PHP_EOL;
104114
$message .= $errMessage;
105115
$context = ['url' => $authUrl];
106-
throw new TestFrameworkException($message, $context);
116+
throw new FastFailException($message, $context);
107117
}
108118
}

‎src/Magento/FunctionalTestingFramework/DataTransport/WebApiExecutor.php

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
namespace Magento\FunctionalTestingFramework\DataTransport;
88

9+
use Magento\FunctionalTestingFramework\Exceptions\FastFailException;
910
use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
1011
use Magento\FunctionalTestingFramework\Util\MftfGlobals;
1112
use Magento\FunctionalTestingFramework\DataTransport\Protocol\CurlInterface;
@@ -45,7 +46,7 @@ class WebApiExecutor implements CurlInterface
4546
* WebApiExecutor Constructor
4647
*
4748
* @param string $storeCode
48-
* @throws TestFrameworkException
49+
* @throws FastFailException
4950
*/
5051
public function __construct($storeCode = null)
5152
{
@@ -58,7 +59,7 @@ public function __construct($storeCode = null)
5859
* Acquire and store the authorization token needed for REST requests
5960
*
6061
* @return void
61-
* @throws TestFrameworkException
62+
* @throws FastFailException
6263
*/
6364
protected function authorize()
6465
{

‎src/Magento/FunctionalTestingFramework/Suite/Handlers/SuiteObjectHandler.php

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,12 @@ public function getAllTestReferences(): array
126126
*/
127127
private function initSuiteData()
128128
{
129-
$suiteDataParser = ObjectManagerFactory::getObjectManager()->create(SuiteDataParser::class);
129+
try {
130+
$suiteDataParser = ObjectManagerFactory::getObjectManager()->create(SuiteDataParser::class);
131+
} catch (\Exception $e) {
132+
throw new FastFailException("Suite Data Parser Error: " . $e->getMessage());
133+
}
134+
130135
$suiteObjectExtractor = new SuiteObjectExtractor();
131136
$this->suiteObjects = $suiteObjectExtractor->parseSuiteDataIntoObjects($suiteDataParser->readSuiteData());
132137
}

‎src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php

Lines changed: 39 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -102,28 +102,37 @@ public function getObject($testName)
102102
*/
103103
public function getAllObjects()
104104
{
105+
$errCount = 0;
105106
$testObjects = [];
106107
foreach ($this->tests as $testName => $test) {
107108
try {
108109
$testObjects[$testName] = $this->extendTest($test);
109110
} catch (FastFailException $exception) {
110111
throw $exception;
111112
} catch (\Exception $exception) {
113+
$errCount++;
112114
LoggingUtil::getInstance()->getLogger(self::class)->error(
113-
"Unable to create test " . $testName . "\n" . $exception->getMessage()
115+
"Unable to extend test " . $testName . "\n" . $exception->getMessage()
114116
);
115-
if (MftfApplicationConfig::getConfig()->getPhase() == MftfApplicationConfig::GENERATION_PHASE) {
116-
print("ERROR: Unable to create test " . $testName . "\n" . $exception->getMessage());
117-
}
118117
if (MftfApplicationConfig::getConfig()->getPhase() != MftfApplicationConfig::EXECUTION_PHASE) {
119118
GenerationErrorHandler::getInstance()->addError(
120119
'test',
121120
$testName,
122-
self::class . ': Unable to create test ' . $exception->getMessage()
121+
self::class . ': Unable to extend test ' . $exception->getMessage()
123122
);
124123
}
125124
}
126125
}
126+
127+
if ($errCount > 0
128+
&& MftfApplicationConfig::getConfig()->getPhase() == MftfApplicationConfig::GENERATION_PHASE) {
129+
print(
130+
"ERROR in TestObjectHandler::getAllObjects(): "
131+
. strval($errCount)
132+
. " Test(s) cannot to be extended. See mftf.log for details."
133+
);
134+
}
135+
127136
return $testObjects;
128137
}
129138

@@ -137,6 +146,7 @@ public function getAllObjects()
137146
*/
138147
public function getTestsByGroup($groupName)
139148
{
149+
$errCount = 0;
140150
$relevantTests = [];
141151
foreach ($this->tests as $test) {
142152
try {
@@ -147,14 +157,12 @@ public function getTestsByGroup($groupName)
147157
} catch (FastFailException $exception) {
148158
throw $exception;
149159
} catch (\Exception $exception) {
160+
$errCount++;
150161
$message = "Unable to reference test "
151162
. $test->getName()
152163
. " for group {$groupName}\n"
153164
. $exception->getMessage();
154165
LoggingUtil::getInstance()->getLogger(self::class)->error($message);
155-
if (MftfApplicationConfig::getConfig()->getPhase() == MftfApplicationConfig::GENERATION_PHASE) {
156-
print('ERROR: ' . $message);
157-
}
158166
if (MftfApplicationConfig::getConfig()->getPhase() != MftfApplicationConfig::EXECUTION_PHASE) {
159167
GenerationErrorHandler::getInstance()->addError(
160168
'test',
@@ -165,6 +173,15 @@ public function getTestsByGroup($groupName)
165173
}
166174
}
167175

176+
if ($errCount > 0
177+
&& MftfApplicationConfig::getConfig()->getPhase() == MftfApplicationConfig::GENERATION_PHASE) {
178+
print(
179+
"ERROR in TestObjectHandler::getTestsByGroup(): "
180+
. strval($errCount)
181+
. " Test(s) cannot be referenced for group {$groupName}. See mftf.log for details."
182+
);
183+
}
184+
168185
return $relevantTests;
169186
}
170187

@@ -194,16 +211,24 @@ public function sanitizeTests($testsToRemove)
194211
*/
195212
private function initTestData($validateAnnotations = true)
196213
{
197-
$testDataParser = ObjectManagerFactory::getObjectManager()->create(TestDataParser::class);
198-
$parsedTestArray = $testDataParser->readTestData();
214+
$parserErrorMessage = null;
215+
try {
216+
$testDataParser = ObjectManagerFactory::getObjectManager()->create(TestDataParser::class);
217+
$parsedTestArray = $testDataParser->readTestData();
199218

200-
$testObjectExtractor = new TestObjectExtractor();
219+
if (!$parsedTestArray) {
220+
$parserErrorMessage = "Could not parse any test in xml.";
221+
}
222+
} catch (\Exception $e) {
223+
$parserErrorMessage = $e->getMessage();
224+
}
201225

202-
if (!$parsedTestArray) {
203-
trigger_error("Could not parse any test in xml.", E_USER_NOTICE);
204-
return;
226+
if ($parserErrorMessage) {
227+
throw new FastFailException("Test Data Parser Error: " . $parserErrorMessage);
205228
}
206229

230+
$testObjectExtractor = new TestObjectExtractor();
231+
207232
$testNameValidator = new NameValidationUtil();
208233
foreach ($parsedTestArray as $testName => $testData) {
209234
try {

‎src/Magento/FunctionalTestingFramework/Util/GenerationErrorHandler.php

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,8 @@ public function printErrorSummary()
113113
$totalErrors = count($this->getErrorsByType($type));
114114
$totalAnnotationErrors = 0;
115115
foreach ($this->getErrorsByType($type) as $entity => $error) {
116-
if ($error['generated'] == true) {
116+
if ( (is_array($error['generated']) && $error['generated'][0] === true)
117+
|| ($error['generated'] === true) ) {
117118
$totalAnnotationErrors++;
118119
}
119120
}
@@ -125,7 +126,7 @@ public function printErrorSummary()
125126
. strval($totalNotGenErrors)
126127
. ' '
127128
. ucfirst($type)
128-
. " failed to generate. See mftf.log for details."
129+
. "(s) failed to generate. See mftf.log for details."
129130
);
130131
}
131132
if ($totalAnnotationErrors > 0) {
@@ -135,7 +136,7 @@ public function printErrorSummary()
135136
. strval($totalAnnotationErrors)
136137
. ' '
137138
. ucfirst($type)
138-
. " generated with annotation errors. See mftf.log for details."
139+
. "(s) generated with annotation errors. See mftf.log for details."
139140
);
140141
}
141142
}

0 commit comments

Comments
 (0)