Skip to content

Commit b27e2a9

Browse files
KevinBKozanKevinBKozan
committed
MQE-1582: Enable Testers To Run Skipped Tests
- added --allowSkipped flag, added to base generate command - moved --force flag to base generate command - fixed some existing issues around not all BaseGenerate inheriting commands not making proper use of --force flag
1 parent 3a73b31 commit b27e2a9

File tree

7 files changed

+65
-31
lines changed

7 files changed

+65
-31
lines changed

‎src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php

Lines changed: 30 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@
99

1010
class MftfApplicationConfig
1111
{
12+
/**
13+
* MFTF Execution Phases
14+
*/
1215
const GENERATION_PHASE = "generation";
1316
const EXECUTION_PHASE = "execution";
1417
const UNIT_TEST_PHASE = "testing";
@@ -50,6 +53,12 @@ class MftfApplicationConfig
5053
*/
5154
private $debugLevel;
5255

56+
/**
57+
* Boolean which allows MFTF to fully generate skipped tests
58+
* @var bool
59+
*/
60+
private $allowSkipped;
61+
5362
/**
5463
* MftfApplicationConfig Singelton Instance
5564
*
@@ -64,13 +73,15 @@ class MftfApplicationConfig
6473
* @param string $phase
6574
* @param boolean $verboseEnabled
6675
* @param string $debugLevel
76+
* @param boolean $allowSkipped
6777
* @throws TestFrameworkException
6878
*/
6979
private function __construct(
7080
$forceGenerate = false,
7181
$phase = self::EXECUTION_PHASE,
7282
$verboseEnabled = null,
73-
$debugLevel = self::LEVEL_NONE
83+
$debugLevel = self::LEVEL_NONE,
84+
$allowSkipped = false
7485
) {
7586
$this->forceGenerate = $forceGenerate;
7687

@@ -89,30 +100,34 @@ private function __construct(
89100
default:
90101
$this->debugLevel = self::LEVEL_DEVELOPER;
91102
}
103+
$this->allowSkipped = $allowSkipped;
92104
}
93105

94106
/**
95107
* Creates an instance of the configuration instance for reference once application has started. This function
96108
* returns void and is only run once during the lifetime of the application.
97109
*
98110
* @param boolean $forceGenerate
99-
* @param string $phase
111+
* @param string $phase
100112
* @param boolean $verboseEnabled
101-
* @param string $debugLevel
113+
* @param string $debugLevel
114+
* @param boolean $allowSkipped
102115
* @return void
116+
* @throws TestFrameworkException
103117
*/
104-
public static function create($forceGenerate, $phase, $verboseEnabled, $debugLevel)
118+
public static function create($forceGenerate, $phase, $verboseEnabled, $debugLevel, $allowSkipped)
105119
{
106120
if (self::$MFTF_APPLICATION_CONTEXT == null) {
107121
self::$MFTF_APPLICATION_CONTEXT =
108-
new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugLevel);
122+
new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugLevel, $allowSkipped);
109123
}
110124
}
111125

112126
/**
113127
* This function returns an instance of the MftfApplicationConfig which is created once the application starts.
114128
*
115129
* @return MftfApplicationConfig
130+
* @throws TestFrameworkException
116131
*/
117132
public static function getConfig()
118133
{
@@ -157,6 +172,16 @@ public function getDebugLevel()
157172
return $this->debugLevel ?? getenv('MFTF_DEBUG');
158173
}
159174

175+
/**
176+
* Returns a boolean indicating whether mftf is generating skipped tests.
177+
*
178+
* @return boolean
179+
*/
180+
public function allowSkipped()
181+
{
182+
return $this->allowSkipped ?? getenv('ALLOW_SKIPPED');
183+
}
184+
160185
/**
161186
* Returns a string which indicates the phase of mftf execution.
162187
*

‎src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,16 @@ protected function configure()
2828
'r',
2929
InputOption::VALUE_NONE,
3030
'remove previous generated suites and tests'
31+
)->addOption(
32+
"force",
33+
'f',
34+
InputOption::VALUE_NONE,
35+
'force generation of tests regardless of Magento Instance Configuration'
36+
)->addOption(
37+
"allowSkipped",
38+
'a',
39+
InputOption::VALUE_NONE,
40+
'Allows MFTF to generate and run skipped tests.'
3141
);
3242
}
3343

‎src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,6 @@ protected function configure()
3636
'name(s) of specific tests to generate'
3737
)->addOption("config", 'c', InputOption::VALUE_REQUIRED, 'default, singleRun, or parallel', 'default')
3838
->addOption(
39-
"force",
40-
'f',
41-
InputOption::VALUE_NONE,
42-
'Force generation of tests regardless of Magento Instance Configuration'
43-
)->addOption(
4439
'time',
4540
'i',
4641
InputOption::VALUE_REQUIRED,
@@ -83,6 +78,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
8378
$debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility
8479
$remove = $input->getOption('remove');
8580
$verbose = $output->isVerbose();
81+
$allowSkipped = $input->getOption('allowSkipped');
8682

8783
if ($json !== null && !json_decode($json)) {
8884
// stop execution if we have failed to properly parse any json passed in by the user
@@ -100,7 +96,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
10096
($debug !== MftfApplicationConfig::LEVEL_NONE));
10197
}
10298

103-
$testConfiguration = $this->createTestConfiguration($json, $tests, $force, $debug, $verbose);
99+
$testConfiguration = $this->createTestConfiguration($json, $tests, $force, $debug, $verbose, $allowSkipped);
104100

105101
// create our manifest file here
106102
$testManifest = TestManifestFactory::makeManifest($config, $testConfiguration['suites']);
@@ -128,18 +124,20 @@ protected function execute(InputInterface $input, OutputInterface $output)
128124
* @param boolean $force
129125
* @param string $debug
130126
* @param boolean $verbose
127+
* @param boolean $allowSkipped
131128
* @return array
132129
* @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
133130
* @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException
134131
*/
135-
private function createTestConfiguration($json, array $tests, bool $force, $debug, bool $verbose)
132+
private function createTestConfiguration($json, array $tests, bool $force, $debug, bool $verbose, bool $allowSkipped)
136133
{
137134
// set our application configuration so we can references the user options in our framework
138135
MftfApplicationConfig::create(
139136
$force,
140137
MftfApplicationConfig::GENERATION_PHASE,
141138
$verbose,
142-
$debug
139+
$debug,
140+
$allowSkipped
143141
);
144142

145143
$testConfiguration = [];

‎src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,6 @@ protected function configure()
3333
"name of tests to generate and execute"
3434
)->addOption('skip-generate', 'k', InputOption::VALUE_NONE, "skip generation and execute existing test")
3535
->addOption(
36-
"force",
37-
'f',
38-
InputOption::VALUE_NONE,
39-
'force generation of tests regardless of Magento Instance Configuration'
40-
)->addOption(
4136
'debug',
4237
'd',
4338
InputOption::VALUE_OPTIONAL,
@@ -66,6 +61,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int
6661
$force = $input->getOption('force');
6762
$remove = $input->getOption('remove');
6863
$debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility
64+
$allowSkipped = $input->getOption('allowSkipped');
65+
6966

7067
if ($skipGeneration and $remove) {
7168
// "skip-generate" and "remove" options cannot be used at the same time
@@ -83,7 +80,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int
8380
]),
8481
'--force' => $force,
8582
'--remove' => $remove,
86-
'--debug' => $debug
83+
'--debug' => $debug,
84+
'--allowSkipped' => $allowSkipped
8785
];
8886
$command->run(new ArrayInput($args), $output);
8987
}

‎src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -76,13 +76,17 @@ protected function configure()
7676
*/
7777
protected function execute(InputInterface $input, OutputInterface $output): int
7878
{
79+
$force = $input->getOption('force');
7980
$debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility
81+
$allowSkipped = $input->getOption('allowSkipped');
82+
8083
// Create Mftf Configuration
8184
MftfApplicationConfig::create(
82-
false,
85+
$force,
8386
MftfApplicationConfig::GENERATION_PHASE,
8487
false,
85-
$debug
88+
$debug,
89+
$allowSkipped
8690
);
8791

8892
$testConfiguration = $this->getFailedTestList();
@@ -96,9 +100,9 @@ protected function execute(InputInterface $input, OutputInterface $output): int
96100
$args = [
97101
'--tests' => $testConfiguration,
98102
'--remove' => true,
99-
'--debug' => $debug
103+
'--debug' => $debug,
104+
'--allowSkipped' => $allowSkipped
100105
];
101-
102106
$command->run(new ArrayInput($args), $output);
103107

104108
$testManifestList = $this->readTestManifestFile();

‎src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,6 @@ protected function configure()
3434
'k',
3535
InputOption::VALUE_NONE,
3636
"only execute a group of tests without generating from source xml"
37-
)->addOption(
38-
"force",
39-
'f',
40-
InputOption::VALUE_NONE,
41-
'force generation of tests regardless of Magento Instance Configuration'
4237
)->addOption(
4338
'debug',
4439
'd',
@@ -72,6 +67,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
7267
$groups = $input->getArgument('groups');
7368
$remove = $input->getOption('remove');
7469
$debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility
70+
$allowSkipped = $input->getOption('allowSkipped');
7571

7672
if ($skipGeneration and $remove) {
7773
// "skip-generate" and "remove" options cannot be used at the same time
@@ -85,7 +81,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int
8581
$force,
8682
MftfApplicationConfig::GENERATION_PHASE,
8783
false,
88-
$debug
84+
$debug,
85+
$allowSkipped
8986
);
9087

9188
if (!$skipGeneration) {
@@ -95,7 +92,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int
9592
'--tests' => $testConfiguration,
9693
'--force' => $force,
9794
'--remove' => $remove,
98-
'--debug' => $debug
95+
'--debug' => $debug,
96+
'--allowSkipped' => $allowSkipped
9997
];
10098

10199
$command->run(new ArrayInput($args), $output);

‎src/Magento/FunctionalTestingFramework/Util/TestGenerator.php

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
namespace Magento\FunctionalTestingFramework\Util;
88

9+
use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
910
use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore;
1011
use Magento\FunctionalTestingFramework\DataGenerator\Handlers\PersistedObjectHandler;
1112
use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject;
@@ -1604,7 +1605,7 @@ private function generateTestPhp($test)
16041605
$testName = str_replace(' ', '', $testName);
16051606
$testAnnotations = $this->generateAnnotationsPhp($test->getAnnotations(), true);
16061607
$dependencies = 'AcceptanceTester $I';
1607-
if ($test->isSkipped()) {
1608+
if ($test->isSkipped() && !MftfApplicationConfig::getConfig()->allowSkipped()) {
16081609
$skipString = "This test is skipped due to the following issues:\\n";
16091610
$issues = $test->getAnnotations()['skip'] ?? null;
16101611
if (isset($issues)) {

0 commit comments

Comments
 (0)