Skip to content

Commit d6a4e8b

Browse files
jilu1jilu1
committed
MQE-1800: Allow generate:tests to continue running even if one test fails generation
1 parent fca945b commit d6a4e8b

File tree

7 files changed

+158
-25
lines changed

7 files changed

+158
-25
lines changed

‎src/Magento/FunctionalTestingFramework/Config/Reader/Filesystem.php

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
namespace Magento\FunctionalTestingFramework\Config\Reader;
77

88
use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
9+
use Magento\FunctionalTestingFramework\Exceptions\FastFailException;
910
use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil;
1011

1112
/**
@@ -240,7 +241,7 @@ protected function validateSchema($configMerger, $filename = null)
240241
true
241242
);
242243
}
243-
throw new \Exception("Schema validation errors found in xml file(s)" . $filename);
244+
throw new FastFailException("Schema validation errors found in xml file(s)" . $filename);
244245
}
245246
}
246247
}

‎src/Magento/FunctionalTestingFramework/Console/GenerateSuiteCommand.php

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
namespace Magento\FunctionalTestingFramework\Console;
99

1010
use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
11+
use Magento\FunctionalTestingFramework\Exceptions\FastFailException;
1112
use Magento\FunctionalTestingFramework\Suite\SuiteGenerator;
1213
use Symfony\Component\Console\Input\InputArgument;
1314
use Symfony\Component\Console\Input\InputInterface;
@@ -43,6 +44,7 @@ protected function configure()
4344
*/
4445
protected function execute(InputInterface $input, OutputInterface $output)
4546
{
47+
$this->setIOStyle($input, $output);
4648
$force = $input->getOption('force');
4749
$debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility
4850
$remove = $input->getOption('remove');
@@ -69,17 +71,40 @@ protected function execute(InputInterface $input, OutputInterface $output)
6971
foreach ($suites as $suite) {
7072
try {
7173
SuiteGenerator::getInstance()->generateSuite($suite);
74+
} catch (FastFailException $e) {
75+
throw $e;
7276
} catch (\Exception $e) {
7377
$errMessages[] = $e->getMessage();
7478
}
7579
}
7680

7781
if ($this->cmdStatus && empty($errMessages)) {
78-
$output->writeLn("Suites Generated");
82+
$this->ioStyle->text("Suites Generated" . PHP_EOL);
7983
return 0;
8084
} else {
81-
$output->writeLn("Suites Generated (with failures)");
85+
$this->printMessages($errMessages);
86+
$this->ioStyle->text("Suites Generated (with failures)" . PHP_EOL);
8287
return 1;
8388
}
8489
}
90+
91+
/**
92+
* Print messages to console
93+
*
94+
* @param string[] $errMessages
95+
* @return void
96+
*/
97+
private function printMessages($errMessages)
98+
{
99+
if (empty($errMessages)) {
100+
return;
101+
}
102+
103+
// Print error
104+
foreach (array_unique($errMessages) as $errMessage) {
105+
if (!empty(trim($errMessage))) {
106+
$this->ioStyle->error(trim($errMessage));
107+
}
108+
}
109+
}
85110
}

‎src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php

Lines changed: 46 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
namespace Magento\FunctionalTestingFramework\Console;
99

1010
use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
11+
use Magento\FunctionalTestingFramework\Exceptions\FastFailException;
1112
use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
1213
use Magento\FunctionalTestingFramework\Suite\SuiteGenerator;
1314
use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler;
@@ -135,6 +136,8 @@ protected function execute(InputInterface $input, OutputInterface $output)
135136

136137
try {
137138
TestGenerator::getInstance(null, $testConfiguration['tests'])->createAllTestFiles($testManifest);
139+
} catch (FastFailException $e) {
140+
throw $e;
138141
} catch (\Exception $e) {
139142
$errMessages[] = $e->getMessage();
140143
}
@@ -146,31 +149,27 @@ protected function execute(InputInterface $input, OutputInterface $output)
146149

147150
try {
148151
SuiteGenerator::getInstance()->generateAllSuites($testManifest);
152+
} catch (FastFailException $e) {
153+
throw $e;
149154
} catch (\Exception $e) {
150155
$errMessages[] = $e->getMessage();
151156
}
152157

153158
$testManifest->generate();
154-
} catch (\Exception $e) {
155-
$message = $e->getMessage() . PHP_EOL;
156-
$message .= !empty($filters) ? 'Filter(s): ' . implode(', ', $filters) . PHP_EOL : '';
157-
$message .= !empty($tests) ? 'Test name(s): ' . implode(', ', $tests) . PHP_EOL : '';
158-
$message .= !empty($json) && empty($tests) ? 'Test configuration: ' . $json . PHP_EOL : '';
159-
$this->ioStyle->note($message);
160159

161-
return 1;
162-
}
160+
if (empty($errMessages)) {
161+
$this->ioStyle->text("Generate Tests Command Run" . PHP_EOL);
163162

164-
if (!empty($errMessages)) {
165-
foreach (array_unique($errMessages) as $errMessage) {
166-
$output->writeln($errMessage);
167-
$output->writeln("\nGenerate Tests Command Run (with failures)");
163+
return 0;
168164
}
169-
return 1;
170-
} else {
171-
$output->writeln("\nGenerate Tests Command Run");
172-
return 0;
165+
} catch (\Exception $e) {
166+
$errMessages[] = $e->getMessage();
173167
}
168+
169+
$this->printMessages($errMessages, $tests, $filters, $json);
170+
$this->ioStyle->text("Generate Tests Command Run (with failures)". PHP_EOL);
171+
172+
return 1;
174173
}
175174

176175
/**
@@ -228,4 +227,35 @@ private function parseTestsConfigJson($json, array $testConfiguration)
228227
$jsonTestConfiguration['suites'] = $testConfigArray['suites'] ?? null;
229228
return $jsonTestConfiguration;
230229
}
230+
231+
/**
232+
* Print messages to console
233+
*
234+
* @param array $errMessages
235+
* @param array $tests
236+
* @param array $filters
237+
* @param string $json
238+
* @return void
239+
*/
240+
private function printMessages($errMessages, $tests, $filters, $json)
241+
{
242+
if (empty($errMessages)) {
243+
return;
244+
}
245+
246+
// Print error
247+
foreach (array_unique($errMessages) as $errMessage) {
248+
if (!empty(trim($errMessage))) {
249+
$this->ioStyle->error(trim($errMessage));
250+
}
251+
}
252+
253+
// Print notice
254+
$message = !empty($filters) ? 'Filter(s): ' . implode(', ', $filters) . PHP_EOL : '';
255+
$message .= !empty($tests) ? 'Test name(s): ' . implode(', ', $tests) . PHP_EOL : '';
256+
$message .= !empty($json) && empty($tests) ? 'Test configuration: ' . $json . PHP_EOL : '';
257+
if (!empty($message)) {
258+
$this->ioStyle->note($message);
259+
}
260+
}
231261
}

‎src/Magento/FunctionalTestingFramework/Exceptions/FastFailException.php

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
<?php
2+
/**
3+
* Copyright © Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
7+
namespace Magento\FunctionalTestingFramework\Exceptions;
8+
9+
use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil;
10+
11+
/**
12+
* Class FastFailException
13+
*
14+
* This exception type should not be caught and should allow fast fail of current execution
15+
*/
16+
class FastFailException extends \Exception
17+
{
18+
/**
19+
* Exception context
20+
*
21+
* @var array
22+
*/
23+
protected $context;
24+
25+
/**
26+
* FastFailException constructor
27+
*
28+
* @param string $message
29+
* @param array $context
30+
*
31+
* @SuppressWarnings(PHPMD.UnusedLocalVariable)
32+
*/
33+
public function __construct($message, $context = [])
34+
{
35+
list($childClass, $callingClass) = debug_backtrace(false, 2);
36+
LoggingUtil::getInstance()->getLogger($callingClass['class'])->error(
37+
$message,
38+
$context
39+
);
40+
41+
$this->context = $context;
42+
parent::__construct($message);
43+
}
44+
45+
/**
46+
* Return exception context
47+
*
48+
* @return array
49+
*/
50+
public function getContext()
51+
{
52+
return $this->context;
53+
}
54+
}

‎src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
namespace Magento\FunctionalTestingFramework\Suite;
88

99
use Magento\FunctionalTestingFramework\Exceptions\Collector\ExceptionCollector;
10+
use Magento\FunctionalTestingFramework\Exceptions\FastFailException;
1011
use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
1112
use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException;
1213
use Magento\FunctionalTestingFramework\Exceptions\XmlException;
@@ -118,6 +119,8 @@ public function generateAllSuites($testManifest)
118119
if (is_array($firstElement)) {
119120
$this->generateSplitSuiteFromTest($suiteName, $suiteContent);
120121
}
122+
} catch (FastFailException $e) {
123+
throw $e;
121124
} catch (\Exception $e) {
122125
$exceptionCollector->addError(self::class, self::class . ': ' . $e->getMessage());
123126
}
@@ -167,6 +170,8 @@ private function generateSuiteFromTest($suiteName, $tests = [], $originalSuiteNa
167170
foreach ($tests as $testName) {
168171
try {
169172
$relevantTests[$testName] = TestObjectHandler::getInstance()->getObject($testName);
173+
} catch (FastFailException $e) {
174+
throw $e;
170175
} catch (\Exception $e) {
171176
$exceptionCollector->addError(
172177
self::class,
@@ -185,6 +190,8 @@ private function generateSuiteFromTest($suiteName, $tests = [], $originalSuiteNa
185190

186191
try {
187192
$this->generateRelevantGroupTests($suiteName, $relevantTests);
193+
} catch (FastFailException $e) {
194+
throw $e;
188195
} catch (\Exception $e) {
189196
$exceptionCollector->addError(
190197
self::class,
@@ -204,6 +211,8 @@ private function generateSuiteFromTest($suiteName, $tests = [], $originalSuiteNa
204211
"suite generated",
205212
['suite' => $suiteName, 'relative_path' => $relativePath]
206213
);
214+
} catch (FastFailException $e) {
215+
throw $e;
207216
} catch (\Exception $e) {
208217
if (file_exists($fullPath)) {
209218
DirSetupUtil::rmdirRecursive($fullPath);

‎src/Magento/FunctionalTestingFramework/Suite/Util/SuiteObjectExtractor.php

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
namespace Magento\FunctionalTestingFramework\Suite\Util;
77

88
use Exception;
9+
use Magento\FunctionalTestingFramework\Exceptions\FastFailException;
910
use Magento\FunctionalTestingFramework\Exceptions\XmlException;
1011
use Magento\FunctionalTestingFramework\Suite\Objects\SuiteObject;
1112
use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler;
@@ -19,6 +20,12 @@
1920
use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException;
2021
use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
2122

23+
/**
24+
* Class SuiteObjectExtractor
25+
* @package Magento\FunctionalTestingFramework\Suite\Util
26+
*
27+
* @SuppressWarnings(PHPMD.CouplingBetweenObjects)
28+
*/
2229
class SuiteObjectExtractor extends BaseObjectExtractor
2330
{
2431
const SUITE_ROOT_TAG = 'suites';
@@ -70,9 +77,8 @@ public function parseSuiteDataIntoObjects($parsedSuiteData)
7077
continue;
7178
}
7279

80+
$this->validateSuiteName($parsedSuite);
7381
try {
74-
$this->validateSuiteName($parsedSuite);
75-
7682
// extract include and exclude references
7783
$groupTestsToInclude = $parsedSuite[self::INCLUDE_TAG_NAME] ?? [];
7884
$groupTestsToExclude = $parsedSuite[self::EXCLUDE_TAG_NAME] ?? [];
@@ -116,11 +122,13 @@ public function parseSuiteDataIntoObjects($parsedSuiteData)
116122
print($includeMessage);
117123
LoggingUtil::getInstance()->getLogger(self::class)->error($includeMessage);
118124
}
125+
} catch (FastFailException $e) {
126+
throw $e;
119127
} catch (\Exception $e) {
120128
$noError = false;
121129
$suiteSkipped++;
122130
if (MftfApplicationConfig::getConfig()->getPhase() == MftfApplicationConfig::GENERATION_PHASE) {
123-
print("Unable to parse suite \"" . $parsedSuite[self::NAME] . "\"");
131+
print("Unable to parse suite \"" . $parsedSuite[self::NAME] . "\"\n");
124132
LoggingUtil::getInstance()->getLogger(self::class)->error(
125133
"Unable to parse suite \"" . $parsedSuite[self::NAME] . "\"\n" . $e->getMessage()
126134
);
@@ -159,14 +167,14 @@ public function parseSuiteDataIntoObjects($parsedSuiteData)
159167
*
160168
* @param array $parsedSuite
161169
* @return void
162-
* @throws XmlException
170+
* @throws FastFailException
163171
*/
164172
private function validateSuiteName($parsedSuite)
165173
{
166174
//check if name used is using special char or the "default" reserved name
167175
NameValidationUtil::validateName($parsedSuite[self::NAME], 'Suite');
168176
if ($parsedSuite[self::NAME] == 'default') {
169-
throw new XmlException("A Suite can not have the name \"default\"");
177+
throw new FastFailException("A Suite can not have the name \"default\"");
170178
}
171179

172180
$suiteName = $parsedSuite[self::NAME];
@@ -179,7 +187,7 @@ private function validateSuiteName($parsedSuite)
179187
}
180188
$exceptionmessage = "\"Suite names and Group names can not have the same value. \t\n" .
181189
"Suite: \"{$suiteName}\" also exists as a group annotation in: \n{$testGroupConflictsFileNames}";
182-
throw new XmlException($exceptionmessage);
190+
throw new FastFailException($exceptionmessage);
183191
}
184192
}
185193

@@ -281,6 +289,8 @@ private function extractTestObjectsFromSuiteRef($suiteReferences)
281289
);
282290
break;
283291
}
292+
} catch (FastFailException $e) {
293+
throw $e;
284294
} catch (\Exception $e) {
285295
$errCount++;
286296
LoggingUtil::getInstance()->getLogger(self::class)->error(

‎src/Magento/FunctionalTestingFramework/Util/TestGenerator.php

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
1010
use Magento\FunctionalTestingFramework\DataGenerator\Handlers\PersistedObjectHandler;
1111
use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject;
12+
use Magento\FunctionalTestingFramework\Exceptions\FastFailException;
1213
use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
1314
use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException;
1415
use Magento\FunctionalTestingFramework\Exceptions\XmlException;
@@ -348,6 +349,7 @@ private function generateInjectMethod()
348349
* @throws TestFrameworkException
349350
* @throws TestReferenceException
350351
* @throws XmlException
352+
* @throws FastFailException
351353
*/
352354
private function assembleAllTestPhp($testManifest, array $testsToIgnore)
353355
{
@@ -398,6 +400,8 @@ private function assembleAllTestPhp($testManifest, array $testsToIgnore)
398400
if ($testManifest != null) {
399401
$testManifest->addTest($test);
400402
}
403+
} catch (FastFailException $e) {
404+
throw $e;
401405
} catch (\Exception $e) {
402406
$this->notGeneratedTestsArray[] = [$test->getName() => $e->getMessage()];
403407
LoggingUtil::getInstance()->getLogger(self::class)->error(

0 commit comments

Comments
 (0)