Skip to content

Commit 86e644a

Browse files
tomreecetomreece
committed
MQE-942: MFTF Annotation Static Check
1 parent df94717 commit 86e644a

File tree

5 files changed

+237
-11
lines changed

5 files changed

+237
-11
lines changed

‎src/Magento/FunctionalTestingFramework/StaticCheck/AnnotationsCheck.php

Lines changed: 218 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,218 @@
1+
<?php
2+
/**
3+
* Copyright © Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
7+
namespace Magento\FunctionalTestingFramework\StaticCheck;
8+
9+
use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
10+
use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler;
11+
use Magento\FunctionalTestingFramework\Util\Script\ScriptUtil;
12+
use Symfony\Component\Console\Input\InputInterface;
13+
use Exception;
14+
15+
/**
16+
* Class AnnotationsCheck
17+
* @package Magento\FunctionalTestingFramework\StaticCheck
18+
*/
19+
class AnnotationsCheck implements StaticCheckInterface
20+
{
21+
const ERROR_LOG_FILENAME = 'mftf-annotations-static-check';
22+
const ERROR_LOG_MESSAGE = 'MFTF Annotations Static Check';
23+
24+
/**
25+
* Array containing all errors found after running the execute() function.
26+
* @var array
27+
*/
28+
private $errors = [];
29+
30+
/**
31+
* String representing the output summary found after running the execute() function.
32+
* @var string
33+
*/
34+
private $output;
35+
36+
/**
37+
* Array containing
38+
* key = Story appended to Title
39+
* value = test names that have that pair
40+
* @var array
41+
*/
42+
private $storiesTitlePairs = [];
43+
44+
/**
45+
* Array containing
46+
* key = testCaseId appended to Title
47+
* value = test names that have that pair
48+
* @var array
49+
*/
50+
private $testCaseIdTitlePairs = [];
51+
52+
/**
53+
* Validates test annotations
54+
*
55+
* @param InputInterface $input
56+
* @return void
57+
* @throws Exception
58+
*/
59+
public function execute(InputInterface $input)
60+
{
61+
// Set MFTF to the UNIT_TEST_PHASE to mute the default DEPRECATION warnings from the TestObjectHandler.
62+
MftfApplicationConfig::create(
63+
true,
64+
MftfApplicationConfig::UNIT_TEST_PHASE,
65+
false,
66+
MftfApplicationConfig::LEVEL_DEFAULT,
67+
true
68+
);
69+
$allTests = TestObjectHandler::getInstance(false)->getAllObjects();
70+
71+
foreach ($allTests as $test) {
72+
$this->validateRequiredAnnotations($test);
73+
$this->aggregateStoriesTitlePairs($test);
74+
$this->aggregateTestCaseIdTitlePairs($test);
75+
}
76+
77+
$this->validateStoriesTitlePairs();
78+
$this->validateTestCaseIdTitlePairs();
79+
80+
$scriptUtil = new ScriptUtil();
81+
$this->output = $scriptUtil->printErrorsToFile(
82+
$this->errors,
83+
self::ERROR_LOG_FILENAME,
84+
self::ERROR_LOG_MESSAGE
85+
);
86+
}
87+
88+
/**
89+
* Return array containing all errors found after running the execute() function.
90+
* @return array
91+
*/
92+
public function getErrors()
93+
{
94+
return $this->errors;
95+
}
96+
97+
/**
98+
* Return string of a short human readable result of the check. For example: "No Dependency errors found."
99+
* @return string
100+
*/
101+
public function getOutput()
102+
{
103+
return $this->output;
104+
}
105+
106+
/**
107+
* Validates that the test has the following annotations:
108+
* stories
109+
* title
110+
* description
111+
* severity
112+
*
113+
* @param $test
114+
*/
115+
private function validateRequiredAnnotations($test)
116+
{
117+
$annotations = $test->getAnnotations();
118+
$missing = [];
119+
120+
$stories = $annotations['stories'] ?? null;
121+
if ($stories === null) {
122+
$missing[] = "stories";
123+
}
124+
125+
$title = $annotations['title'] ?? null;
126+
if ($title === null) {
127+
$missing[] = "title";
128+
}
129+
130+
$description = $annotations['description']['main'] ?? null;
131+
if ($description === null) {
132+
$missing[] = "description";
133+
}
134+
135+
$severity = $annotations['severity'] ?? null;
136+
if ($severity === null) {
137+
$missing[] = "severity";
138+
}
139+
140+
$allMissing = join(", ", $missing);
141+
if (strlen($allMissing) > 0) {
142+
$this->errors[][] = "Test {$test->getName()} is missing the required annotations: " . $allMissing;
143+
}
144+
}
145+
146+
/**
147+
* Add the key = "stories appended to title", value = test name, to the class variable.
148+
*
149+
* @param $test
150+
*/
151+
private function aggregateStoriesTitlePairs($test)
152+
{
153+
$annotations = $test->getAnnotations();
154+
$stories = $annotations['stories'][0] ?? null;
155+
$title = $this->getTestTitleWithoutPrefix($test);
156+
if ($stories !== null && $title !== null) {
157+
$this->storiesTitlePairs[$stories . $title][] = $test->getName();
158+
}
159+
}
160+
161+
/**
162+
* Add the key = "testCaseId appended to title", value = test name, to the class variable.
163+
*
164+
* @param $test
165+
*/
166+
private function aggregateTestCaseIdTitlePairs($test)
167+
{
168+
$annotations = $test->getAnnotations();
169+
$testCaseId = $annotations['testCaseId'][0] ?? null;
170+
$title = $this->getTestTitleWithoutPrefix($test);
171+
if ($testCaseId !== null && $title !== null) {
172+
$this->testCaseIdTitlePairs[$testCaseId . $title][] = $test->getName();
173+
}
174+
}
175+
176+
/**
177+
* Strip away the testCaseId prefix that was automatically added to the test title
178+
* so that way we have just the raw title from the XML file.
179+
*
180+
* @param $test
181+
* @return string|null
182+
*/
183+
private function getTestTitleWithoutPrefix($test)
184+
{
185+
$annotations = $test->getAnnotations();
186+
$title = $annotations['title'][0] ?? null;
187+
if ($title === null) {
188+
return null;
189+
} else {
190+
$testCaseId = $annotations['testCaseId'][0] ?? "[NO TESTCASEID]";
191+
return substr($title, strlen($testCaseId . ": "));
192+
}
193+
}
194+
195+
/**
196+
* Adds an error if any story+title pairs are used by more than one test.
197+
*/
198+
private function validateStoriesTitlePairs()
199+
{
200+
foreach ($this->storiesTitlePairs as $pair) {
201+
if (sizeof($pair) > 1) {
202+
$this->errors[][] = "Stories + title combination must be unique: " . join(", ", $pair);
203+
}
204+
}
205+
}
206+
207+
/**
208+
* Adds an error if any testCaseId+title pairs are used by more than one test.
209+
*/
210+
private function validateTestCaseIdTitlePairs()
211+
{
212+
foreach ($this->testCaseIdTitlePairs as $pair) {
213+
if (sizeof($pair) > 1) {
214+
$this->errors[][] = "testCaseId + title combination must be unique: " . join(", ", $pair);
215+
}
216+
}
217+
}
218+
}

‎src/Magento/FunctionalTestingFramework/StaticCheck/StaticChecksList.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ public function __construct(array $checks = [])
3333
'testDependencies' => new TestDependencyCheck(),
3434
'actionGroupArguments' => new ActionGroupArgumentsCheck(),
3535
self::DEPRECATED_ENTITY_USAGE_CHECK_NAME => new DeprecatedEntityUsageCheck(),
36+
'annotations' => new AnnotationsCheck()
3637
] + $checks;
3738
}
3839

‎src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -53,11 +53,11 @@ class TestObjectHandler implements ObjectHandlerInterface
5353
* @return TestObjectHandler
5454
* @throws XmlException
5555
*/
56-
public static function getInstance()
56+
public static function getInstance($validateAnnotations = true)
5757
{
5858
if (!self::$testObjectHandler) {
5959
self::$testObjectHandler = new TestObjectHandler();
60-
self::$testObjectHandler->initTestData();
60+
self::$testObjectHandler->initTestData($validateAnnotations);
6161
}
6262

6363
return self::$testObjectHandler;
@@ -129,7 +129,7 @@ public function getTestsByGroup($groupName)
129129
* @SuppressWarnings(PHPMD.UnusedPrivateMethod)
130130
* @throws XmlException
131131
*/
132-
private function initTestData()
132+
private function initTestData($validateAnnotations = true)
133133
{
134134
$testDataParser = ObjectManagerFactory::getObjectManager()->create(TestDataParser::class);
135135
$parsedTestArray = $testDataParser->readTestData();
@@ -150,15 +150,17 @@ private function initTestData()
150150
continue;
151151
}
152152
try {
153-
$this->tests[$testName] = $testObjectExtractor->extractTestData($testData);
153+
$this->tests[$testName] = $testObjectExtractor->extractTestData($testData, $validateAnnotations);
154154
} catch (XmlException $exception) {
155155
$exceptionCollector->addError(self::class, $exception->getMessage());
156156
}
157157
}
158158
$exceptionCollector->throwException();
159159
$testNameValidator->summarize(NameValidationUtil::TEST_NAME);
160-
$testObjectExtractor->getAnnotationExtractor()->validateStoryTitleUniqueness();
161-
$testObjectExtractor->getAnnotationExtractor()->validateTestCaseIdTitleUniqueness();
160+
if ($validateAnnotations) {
161+
$testObjectExtractor->getAnnotationExtractor()->validateStoryTitleUniqueness();
162+
$testObjectExtractor->getAnnotationExtractor()->validateTestCaseIdTitleUniqueness();
163+
}
162164
}
163165

164166
/**

‎src/Magento/FunctionalTestingFramework/Test/Util/AnnotationExtractor.php

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ public function __construct()
6060
* @return array
6161
* @throws XmlException
6262
*/
63-
public function extractAnnotations($testAnnotations, $filename)
63+
public function extractAnnotations($testAnnotations, $filename, $validateAnnotations = true)
6464
{
6565
$annotationObjects = [];
6666
$annotations = $this->stripDescriptorTags($testAnnotations, self::NODE_NAME);
@@ -82,7 +82,9 @@ public function extractAnnotations($testAnnotations, $filename)
8282

8383
if ($annotationKey == "skip") {
8484
$annotationData = $annotationData['issueId'];
85-
$this->validateSkippedIssues($annotationData, $filename);
85+
if ($validateAnnotations) {
86+
$this->validateSkippedIssues($annotationData, $filename);
87+
}
8688
}
8789

8890
foreach ($annotationData as $annotationValue) {
@@ -100,7 +102,9 @@ public function extractAnnotations($testAnnotations, $filename)
100102
}
101103

102104
$this->addTestCaseIdToTitle($annotationObjects, $filename);
103-
$this->validateMissingAnnotations($annotationObjects, $filename);
105+
if ($validateAnnotations) {
106+
$this->validateMissingAnnotations($annotationObjects, $filename);
107+
}
104108
$this->addStoryTitleToMap($annotationObjects, $filename);
105109

106110
return $annotationObjects;

‎src/Magento/FunctionalTestingFramework/Test/Util/TestObjectExtractor.php

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ public function getAnnotationExtractor()
8787
* @return TestObject
8888
* @throws \Exception
8989
*/
90-
public function extractTestData($testData)
90+
public function extractTestData($testData, $validateAnnotations = true)
9191
{
9292
// validate the test name for blacklisted char (will cause allure report issues) MQE-483
9393
NameValidationUtil::validateName($testData[self::NAME], "Test");
@@ -117,7 +117,8 @@ public function extractTestData($testData)
117117

118118
$testAnnotations = $this->annotationExtractor->extractAnnotations(
119119
$testData[self::TEST_ANNOTATIONS] ?? [],
120-
$testData[self::NAME]
120+
$testData[self::NAME],
121+
$validateAnnotations
121122
);
122123

123124
//Override features with module name if present, populates it otherwise

0 commit comments

Comments
 (0)