Skip to content

Commit f7cb05e

Browse files
aljcalandraaljcalandra
committed
MQE-1068: Require Issue ID for Skipped Test
- Added Schema information for Skip and issueId - Added tests around skip code - Added deprecation warning for group skip
1 parent 6281e31 commit f7cb05e

File tree

9 files changed

+193
-2
lines changed

9 files changed

+193
-2
lines changed

‎dev/tests/verification/Resources/SkippedTest.txt

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
<?php
2+
namespace Magento\AcceptanceTest\_default\Backend;
3+
4+
use Magento\FunctionalTestingFramework\AcceptanceTester;
5+
use Magento\FunctionalTestingFramework\DataGenerator\Handlers\DataObjectHandler;
6+
use Magento\FunctionalTestingFramework\DataGenerator\Persist\DataPersistenceHandler;
7+
use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject;
8+
use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore;
9+
use \Codeception\Util\Locator;
10+
use Yandex\Allure\Adapter\Annotation\Features;
11+
use Yandex\Allure\Adapter\Annotation\Stories;
12+
use Yandex\Allure\Adapter\Annotation\Title;
13+
use Yandex\Allure\Adapter\Annotation\Description;
14+
use Yandex\Allure\Adapter\Annotation\Parameter;
15+
use Yandex\Allure\Adapter\Annotation\Severity;
16+
use Yandex\Allure\Adapter\Model\SeverityLevel;
17+
use Yandex\Allure\Adapter\Annotation\TestCaseId;
18+
19+
/**
20+
* @Title("skippedTest")
21+
* @Description("")
22+
* @skipIssueId SkippedValue
23+
*/
24+
class SkippedTestCest
25+
{
26+
/**
27+
* @Stories({"skipped"})
28+
* @Severity(level = SeverityLevel::MINOR)
29+
* @Features({"TestModule"})
30+
* @Parameter(name = "AcceptanceTester", value="$I")
31+
* @param AcceptanceTester $I
32+
* @return void
33+
* @throws \Exception
34+
*/
35+
public function SkippedTest(AcceptanceTester $I, \Codeception\Scenario $scenario)
36+
{
37+
$scenario->skip("This test is skipped");
38+
}
39+
}

‎dev/tests/verification/Resources/SkippedTestTwoIssues.txt

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
<?php
2+
namespace Magento\AcceptanceTest\_default\Backend;
3+
4+
use Magento\FunctionalTestingFramework\AcceptanceTester;
5+
use Magento\FunctionalTestingFramework\DataGenerator\Handlers\DataObjectHandler;
6+
use Magento\FunctionalTestingFramework\DataGenerator\Persist\DataPersistenceHandler;
7+
use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject;
8+
use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore;
9+
use \Codeception\Util\Locator;
10+
use Yandex\Allure\Adapter\Annotation\Features;
11+
use Yandex\Allure\Adapter\Annotation\Stories;
12+
use Yandex\Allure\Adapter\Annotation\Title;
13+
use Yandex\Allure\Adapter\Annotation\Description;
14+
use Yandex\Allure\Adapter\Annotation\Parameter;
15+
use Yandex\Allure\Adapter\Annotation\Severity;
16+
use Yandex\Allure\Adapter\Model\SeverityLevel;
17+
use Yandex\Allure\Adapter\Annotation\TestCaseId;
18+
19+
/**
20+
* @Title("skippedMultipleIssuesTest")
21+
* @Description("")
22+
* @skipIssueId SkippedValue
23+
* @skipIssueId SecondSkippedValue
24+
*/
25+
class SkippedTestTwoIssuesCest
26+
{
27+
/**
28+
* @Stories({"skippedMultiple"})
29+
* @Severity(level = SeverityLevel::MINOR)
30+
* @Features({"TestModule"})
31+
* @Parameter(name = "AcceptanceTester", value="$I")
32+
* @param AcceptanceTester $I
33+
* @return void
34+
* @throws \Exception
35+
*/
36+
public function SkippedTestTwoIssues(AcceptanceTester $I, \Codeception\Scenario $scenario)
37+
{
38+
$scenario->skip("This test is skipped");
39+
}
40+
}

‎dev/tests/verification/TestModule/Test/SkippedTest.xml

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
/**
4+
* Copyright © Magento, Inc. All rights reserved.
5+
* See COPYING.txt for license details.
6+
*/
7+
-->
8+
9+
<tests xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
10+
xsi:noNamespaceSchemaLocation="../../../../../src/Magento/FunctionalTestingFramework/Test/etc/testSchema.xsd">
11+
<test name="SkippedTest">
12+
<annotations>
13+
<stories value="skipped"/>
14+
<title value="skippedTest"/>
15+
<description value=""/>
16+
<severity value="AVERAGE"/>
17+
<skip>
18+
<issueId value="SkippedValue"/>
19+
</skip>
20+
</annotations>
21+
</test>
22+
<test name="SkippedTestTwoIssues">
23+
<annotations>
24+
<stories value="skippedMultiple"/>
25+
<title value="skippedMultipleIssuesTest"/>
26+
<description value=""/>
27+
<severity value="AVERAGE"/>
28+
<skip>
29+
<issueId value="SkippedValue"/>
30+
<issueId value="SecondSkippedValue"/>
31+
</skip>
32+
</annotations>
33+
</test>
34+
</tests>

‎dev/tests/verification/Tests/SkippedGenerationTest.php

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
<?php
2+
/**
3+
* Copyright © Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
namespace tests\verification\Tests;
7+
8+
use tests\util\MftfTestCase;
9+
10+
class SkippedGenerationTest extends MftfTestCase
11+
{
12+
/**
13+
* Tests skipped test generation.
14+
*
15+
* @throws \Exception
16+
* @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
17+
*/
18+
public function testSkippedGeneration()
19+
{
20+
$this->generateAndCompareTest('SkippedTest');
21+
}
22+
23+
/**
24+
* Tests skipped test with multiple issues generation.
25+
*
26+
* @throws \Exception
27+
* @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
28+
*/
29+
public function testMultipleSkippedIssuesGeneration()
30+
{
31+
$this->generateAndCompareTest('SkippedTestTwoIssues');
32+
}
33+
}

‎etc/di.xml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -258,6 +258,7 @@
258258
<item name="/tests/test/annotations/group" xsi:type="string">/tests/test/annotations/group</item>
259259
<item name="/tests/test/annotations/env" xsi:type="string">/tests/test/annotations/env</item>
260260
<item name="/tests/test/annotations/return" xsi:type="string">/tests/test/annotations/return</item>
261+
<item name="/tests/test/annotations/skip/issueId" xsi:type="string">/tests/test/annotations/skip/issueId</item>
261262
</argument>
262263
</arguments>
263264
</virtualType>

‎src/Magento/FunctionalTestingFramework/Test/Objects/TestObject.php

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,11 @@ public function getParentName()
126126
*/
127127
public function isSkipped()
128128
{
129-
if (array_key_exists('group', $this->annotations) && (in_array("skip", $this->annotations['group']))) {
129+
// TODO remove elseif when group skip is no longer allowed
130+
if (array_key_exists('skip', $this->annotations)) {
131+
return true;
132+
} elseif (array_key_exists('group', $this->annotations) && (in_array("skip", $this->annotations['group']))) {
133+
print("Use of group skip will be deprecated in MFTF 3.0.0. Please update tests to use skip tags.");
130134
return true;
131135
}
132136
return false;

‎src/Magento/FunctionalTestingFramework/Test/Util/AnnotationExtractor.php

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ public function __construct()
4444
* @param array $testAnnotations
4545
* @param string $filename
4646
* @return array
47+
* @throws XmlException
4748
*/
4849
public function extractAnnotations($testAnnotations, $filename)
4950
{
@@ -61,10 +62,14 @@ public function extractAnnotations($testAnnotations, $filename)
6162
);
6263
continue;
6364
}
64-
65+
if ($annotationKey == "skip") {
66+
$annotationData = $annotationData['issueId'];
67+
$this->validateSkippedIssues($annotationData, $filename);
68+
}
6569
foreach ($annotationData as $annotationValue) {
6670
$annotationValues[] = $annotationValue[self::ANNOTATION_VALUE];
6771
}
72+
6873
$annotationObjects[$annotationKey] = $annotationValues;
6974
}
7075
$this->addStoryTitleToMap($annotationObjects, $filename);
@@ -113,6 +118,22 @@ public function validateStoryTitleUniqueness()
113118
}
114119
}
115120

121+
/**
122+
* Validates that all issueId tags contain a non-empty value
123+
* @param array $issues
124+
* @param string $filename
125+
* @throws XmlException
126+
* @return void
127+
*/
128+
public function validateSkippedIssues($issues, $filename)
129+
{
130+
foreach ($issues as $issueId)
131+
if (empty($issueId['value'])) {
132+
$message = "issueId for skipped tests cannot be empty. Test: $filename";
133+
throw new XmlException($message);
134+
}
135+
}
136+
116137
/**
117138
* This method transforms Magento severity values from Severity annotation
118139
* Returns Allure annotation value

‎src/Magento/FunctionalTestingFramework/Test/etc/mergedTestSchema.xsd

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
<xs:element type="annotationType" name="testCaseId" minOccurs="0" maxOccurs="unbounded"/>
3030
<xs:element type="annotationType" name="useCaseId" minOccurs="0" maxOccurs="unbounded"/>
3131
<xs:element type="annotationType" name="group" minOccurs="0" maxOccurs="unbounded"/>
32+
<xs:element type="skipType" name="skip" minOccurs="0" maxOccurs="unbounded"/>
3233
<xs:element type="annotationType" name="return" minOccurs="0" maxOccurs="unbounded"/>
3334
</xs:choice>
3435
</xs:complexType>
@@ -55,6 +56,18 @@
5556
</xs:extension>
5657
</xs:simpleContent>
5758
</xs:complexType>
59+
<xs:complexType name="skipType">
60+
<xs:sequence maxOccurs="unbounded">
61+
<xs:element type="issueType" name="issueId" maxOccurs="unbounded"/>
62+
</xs:sequence>
63+
</xs:complexType>
64+
<xs:complexType name="issueType">
65+
<xs:simpleContent>
66+
<xs:extension base="xs:string">
67+
<xs:attribute type="xs:string" name="value" use="required"/>
68+
</xs:extension>
69+
</xs:simpleContent>
70+
</xs:complexType>
5871
<xs:complexType name="hookType">
5972
<xs:choice minOccurs="0" maxOccurs="unbounded">
6073
<xs:group ref="testTypeTags"/>

‎src/Magento/FunctionalTestingFramework/Util/TestGenerator.php

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -437,6 +437,12 @@ private function generateClassAnnotations($annotationType, $annotationName)
437437
$annotationToAppend .= sprintf(" * @group %s\n", $group);
438438
}
439439
break;
440+
441+
case "skip":
442+
foreach ($annotationName as $issue) {
443+
$annotationToAppend .= sprintf(" * @skipIssueId %s\n", $issue);
444+
}
445+
break;
440446
}
441447

442448
return $annotationToAppend;

0 commit comments

Comments
 (0)