Skip to content

Commit 501c65a

Browse files
committed
Merge remote-tracking branch 'origin/develop' into MQE-969
2 parents dfcd5c7 + 241734e commit 501c65a

File tree

25 files changed

+714
-83
lines changed

25 files changed

+714
-83
lines changed

CHANGELOG.md

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,56 @@
11
Magento Functional Testing Framework Changelog
22
================================================
33

4+
2.2.0
5+
-----
6+
### Enhancements
7+
* Traceability
8+
* Javascript errors are now logged and reported in test output.
9+
* Test failures are no longer overwritten by failures in an `<after>` hook.
10+
* Tests will no longer execute an `<after>` hook twice if a failure triggered in the `<after>` hook.
11+
* Tests marked with `<group value="skip">` will now appear in generated Allure reports.
12+
* Along with the above, the `robo group` command no longer omits the `skip` group (skipped tests are picked up but not fully executed).
13+
* Modularity
14+
* MFTF no longer relies on relative pathing to determine its path to tests or Magento (favoring composer information if available).
15+
* Tests and test materials are now read in from Magento modules as well as extensions in addition to `dev/tests/acceptance`.
16+
* See DevDocs `Getting Started` for details on expected paths and merge order.
17+
* Customizability
18+
* Creation of Suites is now supported
19+
* `<suite>` can include tests via `name`, module, or `<group>` tags.
20+
* Consolidation of preconditions can be achieved via use of `<before/after>` tags in a `<suite>`
21+
* All normal test actions are supported
22+
* Data returned from actions is not available for reference in subsequent tests (`createData` or `grab` actions).
23+
* `robo generate:tests` generates all suites and tests, and can be given a JSON configuration to generate specific test/suites.
24+
* See MFTF Devdocs "Suite" page for more details.
25+
* `<deleteData>` may now be called against a `url` instead of a stepKey reference.
26+
* `<dragAndDrop>` may now be given an additional `x/y` offset.
27+
* `<executeJS>` now returns a variable based on what the executed script returns.
28+
* Added `<element>` `type="block"`.
29+
* `<page>` elements may now be blank (contain no child sections).
30+
* Maintainability
31+
* `robo generate:tests --config parallel` now accepts a `--lines` argument, for grouping and sorting based on test length.
32+
* `robo generate:tests` now checks for:
33+
* Duplicate step keys within an `actionGroup`.
34+
* Ambiguous or invalid `stepKey` references (in merge files).
35+
* `robo generate:tests` now suppresses warnings by default. The command now accepts a `--verbose` flag to show full output including warnings.
36+
37+
### Fixes
38+
* Exception message for the `<conditionalClick>` action now correctly references the `selector` given.
39+
* Usage of multiple parameterized elements in a `selector` now correctly resolves all element references.
40+
* Usage of multiple uniqueness references on the same entity now generate correctly.
41+
* Persisted entity references are correctly interpolated with `<page>` url of `type="admin"`.
42+
* Metadata that contains 2 or more params in its `url` now correctly resolve parameters.
43+
* Arguments can now be passed to `x` and `y` attributes in `actionGroup`.
44+
* Arguments can now be passed to nested `<assert*>` action elements.
45+
* The `<seeInField>` action can now be used to assert against empty strings.
46+
* Empty `<data>` elements within an `<entity>` now generate correctly.
47+
* Mapping of the `<magentoCLI>` to the custom command has been fixed.
48+
49+
### GitHub Issues/Pull requests:
50+
* [#89](https://github.com/magento/magento2-functional-testing-framework/pull/89) -- Add ability to use array entities as arguments.
51+
* [#68](https://github.com/magento/magento2-functional-testing-framework/issues/68) -- Excessive double quotes are being generated in WaitForElementChange method arguments (fixed in [#103](https://github.com/magento/magento2-functional-testing-framework/pull/103))
52+
* [#31](https://github.com/magento/magento2-functional-testing-framework/issues/31) -- Can't run tests without a store having "default" store code (fixed in [#86](https://github.com/magento/magento2-functional-testing-framework/pull/86))
53+
454
2.1.2
555
-----
656
### Enhancements

composer.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"name": "magento/magento2-functional-testing-framework",
33
"description": "Magento2 Functional Testing Framework",
44
"type": "library",
5-
"version": "2.1.2",
5+
"version": "2.2.0",
66
"license": "AGPL-3.0",
77
"keywords": ["magento", "automation", "functional", "testing"],
88
"config": {
@@ -17,7 +17,7 @@
1717
"flow/jsonpath": ">0.2",
1818
"fzaninotto/faker": "^1.6",
1919
"mustache/mustache": "~2.5",
20-
"symfony/process": ">=2.7 <3.4",
20+
"symfony/process": "^2.8 || ^3.1 || ^4.0",
2121
"vlucas/phpdotenv": "^2.4"
2222
},
2323
"require-dev": {

composer.lock

Lines changed: 7 additions & 7 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

dev/tests/_bootstrap.php

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,27 @@
6262
require($unitUtilFile);
6363
}
6464

65+
66+
// Mocks suite files location getter return to get files in verification/_suite Directory
67+
// This mocks the paths of the suite files but still parses the xml files
68+
$suiteDirectory = TESTS_BP . DIRECTORY_SEPARATOR . "verification" . DIRECTORY_SEPARATOR . "_suite";
69+
70+
$paths = [
71+
$suiteDirectory . DIRECTORY_SEPARATOR . 'functionalSuite.xml',
72+
$suiteDirectory . DIRECTORY_SEPARATOR . 'functionalSuiteHooks.xml'
73+
];
74+
75+
// create and return the iterator for these file paths
76+
$iterator = new Magento\FunctionalTestingFramework\Util\Iterator\File($paths);
77+
try {
78+
AspectMock\Test::double(
79+
Magento\FunctionalTestingFramework\Config\FileResolver\Root::class,
80+
['get' => $iterator]
81+
)->make();
82+
} catch (Exception $e) {
83+
echo "Suite directory not mocked.";
84+
}
85+
6586
function sortInterfaces($files)
6687
{
6788
$bottom = [];
Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
<?php
2+
/**
3+
* Copyright © Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
namespace Tests\unit\Magento\FunctionalTestFramework\Suite;
7+
8+
use AspectMock\Test as AspectMock;
9+
use Magento\FunctionalTestingFramework\ObjectManager\ObjectManager;
10+
use Magento\FunctionalTestingFramework\ObjectManagerFactory;
11+
use Magento\FunctionalTestingFramework\Suite\SuiteGenerator;
12+
use Magento\FunctionalTestingFramework\Suite\Generators\GroupClassGenerator;
13+
use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler;
14+
use Magento\FunctionalTestingFramework\Suite\Parsers\SuiteDataParser;
15+
use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler;
16+
use Magento\FunctionalTestingFramework\Test\Parsers\TestDataParser;
17+
use Magento\FunctionalTestingFramework\Util\Manifest\DefaultTestManifest;
18+
use PHPUnit\Framework\TestCase;
19+
use tests\unit\Util\SuiteDataArrayBuilder;
20+
use tests\unit\Util\TestDataArrayBuilder;
21+
22+
class SuiteGeneratorTest extends TestCase
23+
{
24+
25+
/**
26+
* Tests generating a single suite given a set of parsed test data
27+
* @throws \Exception
28+
*/
29+
public function testGenerateSuite()
30+
{
31+
$suiteDataArrayBuilder = new SuiteDataArrayBuilder();
32+
$mockData = $suiteDataArrayBuilder
33+
->withName('basicTestSuite')
34+
->withAfterHook()
35+
->withBeforeHook()
36+
->includeTests(['simpleTest'])
37+
->includeGroups(['group1'])
38+
->build();
39+
40+
$testDataArrayBuilder = new TestDataArrayBuilder();
41+
$mockSimpleTest = $testDataArrayBuilder
42+
->withName('simpleTest')
43+
->withTestActions()
44+
->build();
45+
46+
$mockTestData = ['tests' => array_merge($mockSimpleTest)];
47+
$this->setMockTestAndSuiteParserOutput($mockTestData, $mockData);
48+
49+
// parse and generate suite object with mocked data
50+
$mockSuiteGenerator = SuiteGenerator::getInstance();
51+
$mockSuiteGenerator->generateSuite("basicTestSuite");
52+
53+
// assert that expected suite is generated
54+
$this->expectOutputString("Suite basicTestSuite generated to _generated/basicTestSuite." . PHP_EOL);
55+
}
56+
57+
/**
58+
* Tests generating all suites given a set of parsed test data
59+
* @throws \Exception
60+
*/
61+
public function testGenerateAllSuites()
62+
{
63+
$suiteDataArrayBuilder = new SuiteDataArrayBuilder();
64+
$mockData = $suiteDataArrayBuilder
65+
->withName('basicTestSuite')
66+
->withAfterHook()
67+
->withBeforeHook()
68+
->includeTests(['simpleTest'])
69+
->includeGroups(['group1'])
70+
->build();
71+
72+
$testDataArrayBuilder = new TestDataArrayBuilder();
73+
$mockSimpleTest = $testDataArrayBuilder
74+
->withName('simpleTest')
75+
->withTestActions()
76+
->build();
77+
78+
$mockTestData = ['tests' => array_merge($mockSimpleTest)];
79+
$this->setMockTestAndSuiteParserOutput($mockTestData, $mockData);
80+
81+
// parse and retrieve suite object with mocked data
82+
$exampleTestManifest = new DefaultTestManifest([], "sample/path");
83+
$mockSuiteGenerator = SuiteGenerator::getInstance();
84+
$mockSuiteGenerator->generateAllSuites($exampleTestManifest);
85+
86+
// assert that expected suites are generated
87+
$this->expectOutputString("Suite basicTestSuite generated to _generated/basicTestSuite." . PHP_EOL);
88+
}
89+
90+
/**
91+
* Function used to set mock for parser return and force init method to run between tests.
92+
*
93+
* @param array $testData
94+
* @throws \Exception
95+
*/
96+
private function setMockTestAndSuiteParserOutput($testData, $suiteData)
97+
{
98+
$property = new \ReflectionProperty(SuiteGenerator::class, 'SUITE_GENERATOR_INSTANCE');
99+
$property->setAccessible(true);
100+
$property->setValue(null);
101+
102+
// clear test object handler value to inject parsed content
103+
$property = new \ReflectionProperty(TestObjectHandler::class, 'testObjectHandler');
104+
$property->setAccessible(true);
105+
$property->setValue(null);
106+
107+
// clear suite object handler value to inject parsed content
108+
$property = new \ReflectionProperty(SuiteObjectHandler::class, 'SUITE_OBJECT_HANLDER_INSTANCE');
109+
$property->setAccessible(true);
110+
$property->setValue(null);
111+
112+
$mockDataParser = AspectMock::double(TestDataParser::class, ['readTestData' => $testData])->make();
113+
$mockSuiteDataParser = AspectMock::double(SuiteDataParser::class, ['readSuiteData' => $suiteData])->make();
114+
$mockGroupClass = AspectMock::double(
115+
GroupClassGenerator::class,
116+
['generateGroupClass' => 'namespace']
117+
)->make();
118+
$mockSuiteClass = AspectMock::double(SuiteGenerator::class, ['generateRelevantGroupTests' => null])->make();
119+
$instance = AspectMock::double(
120+
ObjectManager::class,
121+
['create' => function ($clazz) use (
122+
$mockDataParser,
123+
$mockSuiteDataParser,
124+
$mockGroupClass,
125+
$mockSuiteClass
126+
) {
127+
if ($clazz == TestDataParser::class) {
128+
return $mockDataParser;
129+
}
130+
if ($clazz == SuiteDataParser::class) {
131+
return $mockSuiteDataParser;
132+
}
133+
if ($clazz == GroupClassGenerator::class) {
134+
return $mockGroupClass;
135+
}
136+
if ($clazz == SuiteGenerator::class) {
137+
return $mockSuiteClass;
138+
}
139+
}]
140+
)->make();
141+
// bypass the private constructor
142+
AspectMock::double(ObjectManagerFactory::class, ['getObjectManager' => $instance]);
143+
144+
$property = new \ReflectionProperty(SuiteGenerator::class, 'groupClassGenerator');
145+
$property->setAccessible(true);
146+
$property->setValue($instance, $instance);
147+
148+
}
149+
}

dev/tests/verification/Resources/BasicFunctionalTest.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ class BasicFunctionalTestCest
9494
$I->dontSeeOptionIsSelected(".functionalTestSelector", "someInput");
9595
$I->doubleClick(".functionalTestSelector");
9696
$I->dragAndDrop(".functionalTestSelector", ".functionalTestSelector2");
97+
$I->dragAndDrop(".functionalTestSelector", ".functionalTestSelector2", 100, 900);
9798
$executeJSKey1 = $I->executeJS("someJSFunction");
9899
$I->fillField(".functionalTestSelector", "someInput");
99100
$I->fillField(".functionalTestSelector", "0");
@@ -112,6 +113,7 @@ class BasicFunctionalTestCest
112113
$I->moveMouseOver(".functionalTestSelector");
113114
$I->openNewTab();
114115
$I->pauseExecution();
116+
$I->performOn("#selector", function(\WebDriverElement $el) {return $el->isDisplayed();});
115117
$I->pressKey("#page", "a");
116118
$I->pressKey("#page", ['ctrl','a'],'new');
117119
$I->pressKey("#page", ['shift','111'],'1','x');
@@ -150,6 +152,7 @@ class BasicFunctionalTestCest
150152
$I->waitForElement(".functionalTestSelector", 30);
151153
$I->waitForElementNotVisible(".functionalTestSelector", 30);
152154
$I->waitForElementVisible(".functionalTestSelector", 30);
155+
$I->waitForElementChange("#selector", function(\WebDriverElement $el) {return $el->isDisplayed();});
153156
$I->waitForJS("someJsFunction", 30);
154157
$I->waitForText("someInput", 30, ".functionalTestSelector");
155158
}

0 commit comments

Comments
 (0)