Skip to content

Commit ec8abe6

Browse files
jilu1jilu1
committed
MQE-1800: Allow generate:tests to continue running even if one test fails generation
1 parent 0cb6d3b commit ec8abe6

File tree

10 files changed

+305
-93
lines changed

10 files changed

+305
-93
lines changed

‎src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,13 @@ class BaseGenerateCommand extends Command
4949
*/
5050
protected $ioStyle = null;
5151

52+
/**
53+
* Command status
54+
*
55+
* @var bool
56+
*/
57+
protected $cmdStatus = true;
58+
5259
/**
5360
* Configures the base command.
5461
*
@@ -111,6 +118,7 @@ protected function getTestAndSuiteConfiguration(array $tests)
111118
$testConfiguration['tests'] = null;
112119
$testConfiguration['suites'] = null;
113120
$testsReferencedInSuites = SuiteObjectHandler::getInstance()->getAllTestReferences();
121+
$this->cmdStatus = SuiteObjectHandler::getInstance()->parseSuccessful();
114122
$suiteToTestPair = [];
115123

116124
foreach($tests as $test) {

‎src/Magento/FunctionalTestingFramework/Console/GenerateSuiteCommand.php

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,13 @@ protected function execute(InputInterface $input, OutputInterface $output)
7272
}
7373
}
7474

75-
$output->writeLn("Suites Generated");
75+
if ($this->cmdStatus) {
76+
$output->writeLn("Suites Generated");
77+
return 0;
78+
} else {
79+
$output->writeLn("Suite parsing error found. See mftf.log for details.");
80+
$output->writeLn("Suites Generated");
81+
return 1;
82+
}
7683
}
7784
}

‎src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -126,20 +126,29 @@ protected function execute(InputInterface $input, OutputInterface $output)
126126
$this->removeGeneratedDirectory($output, $verbose);
127127
}
128128

129+
$errMessages = [];
129130
try {
130131
$testConfiguration = $this->createTestConfiguration($json, $tests);
131132

132133
// create our manifest file here
133134
$testManifest = TestManifestFactory::makeManifest($config, $testConfiguration['suites']);
134135

135-
TestGenerator::getInstance(null, $testConfiguration['tests'])->createAllTestFiles($testManifest);
136+
try {
137+
TestGenerator::getInstance(null, $testConfiguration['tests'])->createAllTestFiles($testManifest);
138+
} catch (\Exception $e) {
139+
$errMessages[] = $e->getMessage();
140+
}
136141

137142
if ($config == 'parallel') {
138143
/** @var ParallelTestManifest $testManifest */
139144
$testManifest->createTestGroups($time);
140145
}
141146

142-
SuiteGenerator::getInstance()->generateAllSuites($testManifest);
147+
try {
148+
SuiteGenerator::getInstance()->generateAllSuites($testManifest);
149+
} catch (\Exception $e) {
150+
$errMessages[] = $e->getMessage();
151+
}
143152

144153
$testManifest->generate();
145154
} catch (\Exception $e) {
@@ -152,7 +161,16 @@ protected function execute(InputInterface $input, OutputInterface $output)
152161
return 1;
153162
}
154163

155-
$output->writeln("Generate Tests Command Run");
164+
if (!empty($errMessages)) {
165+
foreach (array_unique($errMessages) as $errMessage) {
166+
$output->writeln($errMessage);
167+
$output->writeln("\nGenerate Tests Command Run");
168+
}
169+
return 1;
170+
} else {
171+
$output->writeln("\nGenerate Tests Command Run");
172+
return 0;
173+
}
156174
}
157175

158176
/**

‎src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,11 @@ protected function execute(InputInterface $input, OutputInterface $output): int
109109
$this->runTestsInSuite($testConfigArray['suites'], $output);
110110
}
111111

112-
return $this->returnCode;
112+
if ($this->returnCode == 0 && $this->cmdStatus) {
113+
return 0;
114+
} else {
115+
return 1;
116+
}
113117
}
114118

115119
/**

‎src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -99,8 +99,12 @@ protected function execute(InputInterface $input, OutputInterface $output): int
9999
$testConfiguration = $this->getFailedTestList();
100100

101101
if ($testConfiguration === null) {
102-
// no failed tests found, run is a success
103-
return 0;
102+
// no failed tests found
103+
if ($this->cmdStatus) {
104+
return 0;
105+
} else {
106+
return 1;
107+
}
104108
}
105109

106110
$command = $this->getApplication()->find('generate:tests');
@@ -150,7 +154,11 @@ function ($type, $buffer) use ($output) {
150154
$this->writeFailedTestToFile($test, $this->testsFailedFile);
151155
}
152156

153-
return $returnCode;
157+
if ($returnCode == 0 && $this->cmdStatus) {
158+
return 0;
159+
} else {
160+
return 1;
161+
}
154162
}
155163

156164
/**

‎src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,11 @@ function ($type, $buffer) use ($output) {
130130
}
131131
$exitCode = 0;
132132
}
133-
return $exitCode;
133+
134+
if ($exitCode == 0 && $this->cmdStatus) {
135+
return 0;
136+
} else {
137+
return 1;
138+
}
134139
}
135140
}

‎src/Magento/FunctionalTestingFramework/Suite/Handlers/SuiteObjectHandler.php

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,13 @@ class SuiteObjectHandler implements ObjectHandlerInterface
3333
*/
3434
private $suiteObjects;
3535

36+
/**
37+
* If suites parsing are successful
38+
*
39+
* @var bool
40+
*/
41+
private $status;
42+
3643
/**
3744
* Avoids instantiation of SuiteObjectHandler by new.
3845
* @return void
@@ -53,7 +60,6 @@ private function __clone()
5360
* Function to enforce singleton design pattern
5461
*
5562
* @return ObjectHandlerInterface
56-
* @throws XmlException
5763
*/
5864
public static function getInstance(): ObjectHandlerInterface
5965
{
@@ -89,6 +95,16 @@ public function getAllObjects(): array
8995
return $this->suiteObjects;
9096
}
9197

98+
/**
99+
* Return if there is any parsing errors
100+
*
101+
* @return bool
102+
*/
103+
public function parseSuccessful(): bool
104+
{
105+
return $this->status;
106+
}
107+
92108
/**
93109
* Function which return all tests referenced by suites.
94110
*
@@ -114,12 +130,17 @@ public function getAllTestReferences(): array
114130
*
115131
* @return void
116132
* @SuppressWarnings(PHPMD.UnusedPrivateMethod)
117-
* @throws XmlException
118133
*/
119134
private function initSuiteData()
120135
{
121136
$suiteDataParser = ObjectManagerFactory::getObjectManager()->create(SuiteDataParser::class);
122137
$suiteObjectExtractor = new SuiteObjectExtractor();
123-
$this->suiteObjects = $suiteObjectExtractor->parseSuiteDataIntoObjects($suiteDataParser->readSuiteData());
138+
$parsedArray = $suiteObjectExtractor->parseSuiteDataIntoObjects($suiteDataParser->readSuiteData());
139+
$this->suiteObjects = $parsedArray['objects'] ?? [];
140+
if (empty($this->suiteObjects)) {
141+
$this->status = false;
142+
} else {
143+
$this->status = $parsedArray['status'] ?? false;
144+
}
124145
}
125146
}

‎src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php

Lines changed: 59 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
namespace Magento\FunctionalTestingFramework\Suite;
88

9+
use Magento\FunctionalTestingFramework\Exceptions\Collector\ExceptionCollector;
910
use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
1011
use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException;
1112
use Magento\FunctionalTestingFramework\Exceptions\XmlException;
@@ -94,27 +95,34 @@ public function generateAllSuites($testManifest)
9495
{
9596
$suites = $testManifest->getSuiteConfig();
9697

98+
$exceptionCollector = new ExceptionCollector();
9799
foreach ($suites as $suiteName => $suiteContent) {
98-
if (empty($suiteContent)) {
99-
LoggingUtil::getInstance()->getLogger(self::class)->notification(
100-
"Suite '" . $suiteName . "' contains no tests and won't be generated." . PHP_EOL,
101-
[],
102-
true
103-
);
104-
continue;
105-
}
106-
$firstElement = array_values($suiteContent)[0];
100+
try {
101+
if (empty($suiteContent)) {
102+
LoggingUtil::getInstance()->getLogger(self::class)->notification(
103+
"Suite '" . $suiteName . "' contains no tests and won't be generated." . PHP_EOL,
104+
[],
105+
true
106+
);
107+
continue;
108+
}
109+
$firstElement = array_values($suiteContent)[0];
107110

108-
// if the first element is a string we know that we simply have an array of tests
109-
if (is_string($firstElement)) {
110-
$this->generateSuiteFromTest($suiteName, $suiteContent);
111-
}
111+
// if the first element is a string we know that we simply have an array of tests
112+
if (is_string($firstElement)) {
113+
$this->generateSuiteFromTest($suiteName, $suiteContent);
114+
}
112115

113-
// if our first element is an array we know that we have split the suites
114-
if (is_array($firstElement)) {
115-
$this->generateSplitSuiteFromTest($suiteName, $suiteContent);
116+
// if our first element is an array we know that we have split the suites
117+
if (is_array($firstElement)) {
118+
$this->generateSplitSuiteFromTest($suiteName, $suiteContent);
119+
}
120+
} catch (\Exception $e) {
121+
$exceptionCollector->addError(self::class, $e->getMessage());
116122
}
117123
}
124+
// Report failure
125+
$this->throwCollectedExceptions($exceptionCollector);
118126
}
119127

120128
/**
@@ -140,9 +148,7 @@ public function generateSuite($suiteName)
140148
* @param array $tests
141149
* @param string $originalSuiteName
142150
* @return void
143-
* @throws TestReferenceException
144-
* @throws XmlException
145-
* @throws TestFrameworkException
151+
* @throws \Exception
146152
*/
147153
private function generateSuiteFromTest($suiteName, $tests = [], $originalSuiteName = null)
148154
{
@@ -152,10 +158,18 @@ private function generateSuiteFromTest($suiteName, $tests = [], $originalSuiteNa
152158
DirSetupUtil::createGroupDir($fullPath);
153159

154160
$relevantTests = [];
161+
$exceptionCollector = new ExceptionCollector();
155162
if (!empty($tests)) {
156163
$this->validateTestsReferencedInSuite($suiteName, $tests, $originalSuiteName);
157164
foreach ($tests as $testName) {
158-
$relevantTests[$testName] = TestObjectHandler::getInstance()->getObject($testName);
165+
try {
166+
$relevantTests[$testName] = TestObjectHandler::getInstance()->getObject($testName);
167+
} catch (\Exception $e) {
168+
$exceptionCollector->addError(
169+
self::class,
170+
"Unable to find relevant test \"{$testName}\" for suite \"{$suiteName}\"\n" . $e->getMessage()
171+
);
172+
}
159173
}
160174
} else {
161175
$relevantTests = SuiteObjectHandler::getInstance()->getObject($suiteName)->getTests();
@@ -169,6 +183,8 @@ private function generateSuiteFromTest($suiteName, $tests = [], $originalSuiteNa
169183
"suite generated",
170184
['suite' => $suiteName, 'relative_path' => $relativePath]
171185
);
186+
187+
$this->throwCollectedExceptions($exceptionCollector);
172188
}
173189

174190
/**
@@ -371,4 +387,27 @@ private static function getYamlConfigFilePath()
371387
{
372388
return FilePathFormatter::format(TESTS_BP);
373389
}
390+
391+
/**
392+
* Log error and throw collected exceptions
393+
*
394+
* @param ExceptionCollector $exceptionCollector
395+
* @return void
396+
* @throws \Exception
397+
*/
398+
private function throwCollectedExceptions($exceptionCollector)
399+
{
400+
if (!empty($exceptionCollector->getErrors())) {
401+
foreach ($exceptionCollector->getErrors() as $file => $errorMessage) {
402+
if (is_array($errorMessage)) {
403+
foreach (array_unique($errorMessage) as $message) {
404+
LoggingUtil::getInstance()->getLogger(SuiteGenerator::class)->error($message);
405+
}
406+
} else {
407+
LoggingUtil::getInstance()->getLogger(SuiteGenerator::class)->error($errorMessage);
408+
}
409+
}
410+
$exceptionCollector->throwException();
411+
}
412+
}
374413
}

0 commit comments

Comments
 (0)