MQE-1800: Allow generate:tests to continue running even if one test fails generation

Author: jilu1

composer.json

@@ -63,7 +63,7 @@
     },
     "autoload-dev": {
         "psr-4": {
-            "tests\\unit\\": "dev/tests/unit"
+            "tests\\": "dev/tests"
         }
     },
     "scripts": {

dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php

@@ -188,7 +188,13 @@ public function mockHandlers($testArray, $suiteArray)
         $property->setAccessible(true);
         $property->setValue($handler, $testArray);
 
-        AspectMock::double(SuiteObjectHandler::class, ['initSuiteData' => ''])->make();
+        AspectMock::double(
+            SuiteObjectHandler::class,
+            [
+                'initSuiteData' => '',
+                'parseSuccessful' => true,
+            ]
+        )->make();
         $handler = SuiteObjectHandler::getInstance();
         $property = new \ReflectionProperty(SuiteObjectHandler::class, 'suiteObjects');
         $property->setAccessible(true);

dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php

@@ -138,7 +138,7 @@ public function testGenerateEmptySuite()
         $this->setMockTestAndSuiteParserOutput($mockTestData, $mockData);
 
         // set expected error message
-        $this->expectExceptionMessage("Suites must not be empty. Suite: \"basicTestSuite\"");
+        $this->expectExceptionMessage("Suite basicTestSuite is not defined in xml or is invalid");
 
         // parse and generate suite object with mocked data
         $mockSuiteGenerator = SuiteGenerator::getInstance();
@@ -180,7 +180,7 @@ public function testInvalidSuiteTestPair()
         $manifest = TestManifestFactory::makeManifest('default', $suiteConfig);
 
         // Set up Expected Exception
-        $this->expectException(TestReferenceException::class);
+        $this->expectException(\Exception::class);
         $this->expectExceptionMessageMatches('(Suite: "Suite2" Tests: "Test1")');
 
         // parse and generate suite object with mocked data and manifest
@@ -204,7 +204,7 @@ public function testNonExistentSuiteTestPair()
         $manifest = TestManifestFactory::makeManifest('default', $suiteConfig);
 
         // Set up Expected Exception
-        $this->expectException(TestReferenceException::class);
+        $this->expectException(\Exception::class);
         $this->expectExceptionMessageMatches('#Suite3 is not defined#');
 
         // parse and generate suite object with mocked data and manifest

dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php

@@ -9,6 +9,7 @@
 use AspectMock\Test as AspectMock;
 
 use Magento\FunctionalTestingFramework\Filter\FilterList;
+use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler;
 use Magento\FunctionalTestingFramework\Test\Objects\ActionObject;
 use Magento\FunctionalTestingFramework\Test\Objects\TestHookObject;
 use Magento\FunctionalTestingFramework\Test\Objects\TestObject;
@@ -41,13 +42,13 @@ public function testEntityException()
 
         $testObject = new TestObject("sampleTest", ["merge123" => $actionObject], [], [], "filename");
 
+        AspectMock::double(TestObjectHandler::class, ['initTestData' => '']);
+
         $testGeneratorObject = TestGenerator::getInstance("", ["sampleTest" => $testObject]);
 
         AspectMock::double(TestGenerator::class, ['loadAllTestObjects' => ["sampleTest" => $testObject]]);
 
-        $this->expectExceptionMessage("Could not resolve entity reference \"{{someEntity.entity}}\" " .
-            "in Action with stepKey \"fakeAction\".\n" .
-            "Exception occurred parsing action at StepKey \"fakeAction\" in Test \"sampleTest\"");
+        $this->expectExceptionMessage("ERROR: 1 Test failed to generate.");
 
         $testGeneratorObject->createAllTestFiles(null, []);
     }

dev/tests/unit/Util/TestLoggingUtil.php

@@ -93,7 +93,7 @@ public function validateMockLogStatmentRegex($type, $regex, $context)
         $records = $this->testLogHandler->getRecords();
         $record = $records[count($records)-1]; // we assume the latest record is what requires validation
         $this->assertEquals(strtoupper($type), $record['level_name']);
-        $this->assertRegExp($regex, $record['message']);
+        $this->assertMatchesRegularExpression($regex, $record['message']);
         $this->assertEquals($context, $record['context']);
     }
 

dev/tests/util/MftfTestCase.php

@@ -80,6 +80,28 @@ public function validateSchemaErrorWithTest($fileContents, $objectType ,$expecte
         }
     }
 
+    /**
+     * Asserts that the given callback throws the given exception
+     *
+     * @param string $expectClass
+     * @param array $expectedMessages
+     * @param callable $callback
+     */
+    protected function assertExceptionRegex(string $expectClass, array $expectedMessages, callable $callback)
+    {
+        try {
+            $callback();
+        } catch (\Throwable $exception) {
+            $this->assertInstanceOf($expectClass, $exception, 'An invalid exception was thrown.');
+            foreach ($expectedMessages as $expectedMessage) {
+                $this->assertMatchesRegularExpression($expectedMessage, $exception->getMessage());
+            }
+            return;
+        }
+
+        $this->fail('No exception was thrown.');
+    }
+
     /**
      * Clears test handler and object manager to force recollection of test data
      *

dev/tests/verification/ResilientGenerationModule/ActionGroup/BasicActionGroup/NotGenerateArgActionGroup.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ /**
+  * Copyright © Magento, Inc. All rights reserved.
+  * See COPYING.txt for license details.
+  */
+-->
+<actionGroups xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/actionGroupSchema.xsd">
+    <actionGroup name="NotGenerateArgActionGroup">
+        <arguments>
+            <argument name="someArgument" defaultValue="ReplacementPerson"/>
+        </arguments>
+        <see selector="{{SampleSection.threeParamElement(someArgument.firstname, someArgument.lastname, 'test')}}" userInput="{{someArgument.lastname}}" stepKey="seeLastName" />
+    </actionGroup>
+</actionGroups>

dev/tests/verification/ResilientGenerationModule/ActionGroup/BasicActionGroup/NotGenerateCreateDataActionGroup.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ /**
+  * Copyright © Magento, Inc. All rights reserved.
+  * See COPYING.txt for license details.
+  */
+-->
+<actionGroups xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/actionGroupSchema.xsd">
+    <actionGroup name="NotGenerateCreateDataActionGroup">
+        <createData entity="NotExtendParentData" stepKey="createData"/>
+    </actionGroup>
+</actionGroups>

dev/tests/verification/ResilientGenerationModule/ActionGroup/BasicActionGroup/NotGenerateExtendChildActionGroup.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ /**
+  * Copyright © Magento, Inc. All rights reserved.
+  * See COPYING.txt for license details.
+  */
+-->
+<actionGroups xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/actionGroupSchema.xsd">
+    <actionGroup name="NotGenerateExtendChildActionGroup" extends="NotGenerateExtendParentActionGroup">
+        <arguments>
+            <argument name="otherCount" type="string"/>
+        </arguments>
+        <grabMultiple selector="notASelector" stepKey="grabProducts"/>
+        <comment userInput="New Input After" stepKey="afterGrabProducts" after="grabProducts"/>
+        <comment userInput="New Input Before" stepKey="beforeGrabProducts" before="grabProducts"/>
+        <assertCount stepKey="assertSecondCount">
+            <expectedResult type="int">{{otherCount}}</expectedResult>
+            <actualResult type="variable">grabProducts</actualResult>
+        </assertCount>
+    </actionGroup>
+</actionGroups>

dev/tests/verification/ResilientGenerationModule/ActionGroup/BasicActionGroup/NotGenerateExtendChildBadReferenceActionGroup.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ /**
+  * Copyright © Magento, Inc. All rights reserved.
+  * See COPYING.txt for license details.
+  */
+-->
+<actionGroups xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/actionGroupSchema.xsd">
+    <actionGroup name="NotGenerateExtendChildBadReferenceActionGroup" extends="extendBasicActionGroup">
+        <fillField stepKey="fill" selector="$name" userInput="$$N.N$$"/>
+    </actionGroup>
+</actionGroups>