MQE-2110: MFTF command to pause test execution

Author: jilu1

docs/commands/mftf.md

@@ -559,6 +559,31 @@ To upgrade all test components inside the `Catalog` module:
 vendor/bin/mftf upgrade:tests /Users/user/magento2/app/code/Magento/Catalog/Test/Mftf/
 ```
 
+### `codecept:run`
+
+MFTF wrapper command invokes `vendor/bin/codecept run`. This command runs tests in functional suite. It does not generate tests for you. You must do that first.
+
+#### Usage
+
+See https://codeception.com/docs/reference/Commands#Run
+
+```bash
+vendor/bin/mftf codecept:run [<suite|test>] --[<option(s)>]
+```
+
+#### Examples
+
+```bash
+# Run all tests in functional suite
+vendor/bin/mftf codecept:run functional
+# Run all tests in functional suite with options
+vendor/bin/mftf codecept:run functional --verbose --steps --debug
+# Run one test
+vendor/bin/mftf codecept:run functional Magento/_generated/default/AdminLoginSuccessfulTestCest
+# Run all tests in default group
+vendor/bin/mftf codecept:run functional --verbose --steps -g default
+```
+
 <!-- LINK DEFINITIONS -->
 
 [configuration]: ../configuration.md

docs/interactive-pause.md

@@ -0,0 +1,58 @@
+# Interactive Pause
+
+It’s hard to write a complete test at once. You will need to try different commands with different arguments before you find a correct path.
+
+Since Codeception 3.0 you can pause execution in any point and enter interactive shell where you will be able to try commands in action. 
+
+Now this `Interactive Pause` feature is available in MFTF and all you need to do is to set `ENABLE_PAUSE=true` in `.env`.
+
+Check it out at [Codeception website][] for documentation and a video to see `Interactive Pause` in action.
+ 
+In short, when a test gets to `$I->pause()` step, it stops and shows a console where you can try all available commands with auto-completion, stash commands, and save screenshot, etc. 
+
+## Generation Time
+
+A `<pause>` action in xml will always be generated into php regardless if `ENABLE_PAUSE=true` is set or not. 
+However, when `ENABLE_PAUSE=true` is set, an additional`pause()` action will be generated in `_failed()` hook for a test,
+so that the test could pause on failure at run time.
+
+## Execution Time
+
+To use `Interactive Pause` at run time, there are two types of MFTF commands to use.
+
+### MFTF Run Commands
+
+When `ENABLE_PAUSE=true` is set, the following MFTF run commands support `Interactive Pause`.
+
+```bash
+vendor/bin/mftf run:group
+```
+
+```bash
+vendor/bin/mftf run:test
+```
+
+```bash
+vendor/bin/mftf run:manifest
+```
+
+```bash
+vendor/bin/mftf run:failed
+```
+
+<div class="bs-callout-warning">
+Note: MFTF run command's `--debug` option is different from Codeception `--debug` mode option. 
+</div>
+
+### MFTF Codecept Run Command
+
+You can also use MFTF's wrapper command to run Codeception directly and activate `Interactive Pause` by passing `--debug` option. 
+You don't need to set `ENABLE_PAUSE=true` for this command.
+
+```bash
+vendor/bin/mftf codecept:run --debug
+```
+
+## References
+
+[Codeception website](https://codeception.com/docs/02-GettingStarted#Interactive-Pause)

src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php

@@ -31,7 +31,8 @@
 class BaseGenerateCommand extends Command
 {
     const MFTF_NOTICES = "Placeholder text for MFTF notices\n";
-    const CODECEPT_RUN_COMMAND = 'codecept:run functional ';
+    const CODECEPT_RUN = 'codecept:run ';
+    const CODECEPT_RUN_FUNCTIONAL = self::CODECEPT_RUN . 'functional ';
 
     /**
      * Enable pause()
@@ -254,15 +255,15 @@ protected function pauseEnabled()
     /**
      * Runs the bin/mftf codecept:run command and returns exit code
      *
-     * @param string          $command
+     * @param string          $commandStr
      * @param OutputInterface $output
      * @return integer
      * @throws \Exception
      */
-    protected function codeceptRunTest(string $command, OutputInterface $output)
+    protected function codeceptRunTest(string $commandStr, OutputInterface $output)
     {
-        $input = new StringInput($command);
-        $command = $this->getApplication()->find('codecept:run');
+        $input = new StringInput($commandStr);
+        $command = $this->getApplication()->find(self::CODECEPT_RUN);
         return $command->run($input, $output);
     }
 }

src/Magento/FunctionalTestingFramework/Console/CodeceptRunCommand.php

@@ -24,7 +24,7 @@ protected function configure()
     {
         $this->setName('codecept:run')
             ->setDescription(
-                "Wrapper command to codecept:run. See https://codeception.com/docs/reference/Commands"
+                "Wrapper command to vendor/bin/codecept:run. See https://codeception.com/docs/reference/Commands#Run"
             );
 
         parent::configure();

src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php

@@ -111,8 +111,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int
     private function runManifestLine(string $manifestLine, OutputInterface $output)
     {
         if (getenv('ENABLE_PAUSE') === 'true') {
-            $codeceptionCommand = BaseGenerateCommand::CODECEPT_RUN_COMMAND
-                . '--verbose --steps --debug' . $manifestLine;
+            $codeceptionCommand = BaseGenerateCommand::CODECEPT_RUN_FUNCTIONAL
+                . '--verbose --steps --debug ' . $manifestLine;
             $input = new StringInput($codeceptionCommand);
             $command = $this->getApplication()->find('codecept:run');
             $subReturnCode = $command->run($input, $output);

src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php

@@ -124,7 +124,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
     private function runTests(array $tests, OutputInterface $output)
     {
         if ($this->pauseEnabled()) {
-            $codeceptionCommand = self::CODECEPT_RUN_COMMAND;
+            $codeceptionCommand = self::CODECEPT_RUN_FUNCTIONAL;
         } else {
             $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional ';
         }
@@ -164,7 +164,7 @@ private function runTests(array $tests, OutputInterface $output)
     private function runTestsInSuite(array $suitesConfig, OutputInterface $output)
     {
         if ($this->pauseEnabled()) {
-            $codeceptionCommand = self::CODECEPT_RUN_COMMAND . '--verbose --steps --debug';
+            $codeceptionCommand = self::CODECEPT_RUN_FUNCTIONAL . '--verbose --steps --debug';
         } else {
             $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept')
                 . ' run functional --verbose --steps ';

src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php

@@ -118,8 +118,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         $returnCode = 0;
         foreach ($testManifestList as $testCommand) {
             if ($this->pauseEnabled()) {
-                $codeceptionCommand = self::CODECEPT_RUN_COMMAND . $testCommand . ' --debug';
-                $this->codeceptRunTest($codeceptionCommand, $output);
+                $codeceptionCommand = self::CODECEPT_RUN_FUNCTIONAL . $testCommand . ' --debug';
+                $returnCode = $this->codeceptRunTest($codeceptionCommand, $output);
             } else {
                 $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional ';
                 $codeceptionCommand .= $testCommand;

src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php

@@ -95,7 +95,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         }
 
         if ($this->pauseEnabled()) {
-            $commandString = self::CODECEPT_RUN_COMMAND . '--verbose --steps --debug';
+            $commandString = self::CODECEPT_RUN_FUNCTIONAL . '--verbose --steps --debug';
         } else {
             $commandString = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional --verbose --steps';
         }

src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php

@@ -836,6 +836,9 @@ public function _failed(TestInterface $test, $fail)
 
         if ($this->pngReport === null && $this->htmlReport === null) {
             $this->saveScreenshot();
+            if (getenv('ENABLE_PAUSE') === 'true') {
+                $this->pause();
+            }
         }
 
         if ($this->current_test == null) {