MQE-1800: Allow generate:tests to continue running even if one test fails generation

Author: jilu1

dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php

@@ -16,6 +16,7 @@
 use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler;
 use Magento\FunctionalTestingFramework\Test\Util\TestObjectExtractor;
 use Magento\FunctionalTestingFramework\Test\Parsers\TestDataParser;
+use Magento\FunctionalTestingFramework\Util\GenerationErrorHandler;
 use Magento\FunctionalTestingFramework\Util\Manifest\DefaultTestManifest;
 use tests\unit\Util\MagentoTestCase;
 use Magento\FunctionalTestingFramework\Util\Manifest\TestManifestFactory;
@@ -179,13 +180,15 @@ public function testInvalidSuiteTestPair()
         $suiteConfig = ['Suite2' => ['Test1']];
         $manifest = TestManifestFactory::makeManifest('default', $suiteConfig);
 
-        // Set up Expected Exception
-        $this->expectException(\Exception::class);
-        $this->expectExceptionMessageMatches('(Suite: "Suite2" Tests: "Test1")');
-
         // parse and generate suite object with mocked data and manifest
         $mockSuiteGenerator = SuiteGenerator::getInstance();
         $mockSuiteGenerator->generateAllSuites($manifest);
+
+        // assert that no exception for generateAllSuites and suite generation error is stored in GenerationErrorHandler
+        $errMessage = 'Cannot reference tests which are not declared as part of suite (Suite: "Suite2" Tests: "Test1")';
+        TestLoggingUtil::getInstance()->validateMockLogStatement('error', $errMessage, []);
+        $suiteErrors = GenerationErrorHandler::getInstance()->getErrorsByType('suite');
+        $this->assertArrayHasKey('Suite2', $suiteErrors);
     }
 
     public function testNonExistentSuiteTestPair()
@@ -203,13 +206,15 @@ public function testNonExistentSuiteTestPair()
         $suiteConfig = ['Suite3' => ['Test1']];
         $manifest = TestManifestFactory::makeManifest('default', $suiteConfig);
 
-        // Set up Expected Exception
-        $this->expectException(\Exception::class);
-        $this->expectExceptionMessageMatches('#Suite3 is not defined#');
-
         // parse and generate suite object with mocked data and manifest
         $mockSuiteGenerator = SuiteGenerator::getInstance();
         $mockSuiteGenerator->generateAllSuites($manifest);
+
+        // assert that no exception for generateAllSuites and suite generation error is stored in GenerationErrorHandler
+        $errMessage = 'Suite Suite3 is not defined in xml or is invalid.';
+        TestLoggingUtil::getInstance()->validateMockLogStatement('error', $errMessage, []);
+        $suiteErrors = GenerationErrorHandler::getInstance()->getErrorsByType('suite');
+        $this->assertArrayHasKey('Suite3', $suiteErrors);
     }
 
     /**
@@ -271,6 +276,14 @@ private function setMockTestAndSuiteParserOutput($testData, $suiteData)
         $property->setValue($instance, $instance);
     }
 
+    /**
+     * clean up function runs after each test
+     */
+    public function tearDown(): void
+    {
+        GenerationErrorHandler::getInstance()->reset();
+    }
+
     /**
      * clean up function runs after all tests
      */

dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php

@@ -20,9 +20,16 @@
 use tests\unit\Util\ObjectHandlerUtil;
 use tests\unit\Util\TestDataArrayBuilder;
 use tests\unit\Util\MockModuleResolverBuilder;
+use tests\unit\Util\TestLoggingUtil;
+use Magento\FunctionalTestingFramework\Util\GenerationErrorHandler;
 
 class TestObjectHandlerTest extends MagentoTestCase
 {
+    public function setUp(): void
+    {
+        TestLoggingUtil::getInstance()->setMockLoggingUtil();
+    }
+
     /**
      * Basic test to validate array => test object conversion.
      *
@@ -254,10 +261,13 @@ public function testGetAllTestObjectsWithInvalidExtends()
         ObjectHandlerUtil::mockTestObjectHandlerWitData(array_merge($testOne, $testTwo));
 
         $toh = TestObjectHandler::getInstance();
-
-        $this->expectException(\Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException::class);
-        $this->expectExceptionMessage("Mftf Test can not extend from itself: " . "testOne");
         $toh->getAllObjects();
+
+        // assert that no exception for getAllObjects and test generation error is stored in GenerationErrorHandler
+        $errorMessage = '/' . preg_quote("Mftf Test can not extend from itself: " . "testOne") . '/';
+        TestLoggingUtil::getInstance()->validateMockLogStatmentRegex('error', $errorMessage, []);
+        $testErrors = GenerationErrorHandler::getInstance()->getErrorsByType('test');
+        $this->assertArrayHasKey('testOne', $testErrors);
     }
 
     /**
@@ -360,6 +370,8 @@ public function testGetTestObjectWhenEnablePause()
      */
     public function tearDown(): void
     {
+        TestLoggingUtil::getInstance()->clearMockLoggingUtil();
         AspectMock::clean();
+        parent::tearDownAfterClass();
     }
 }

dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/AnnotationExtractorTest.php

@@ -12,6 +12,7 @@
 use Monolog\Logger;
 use PHPUnit\Framework\TestCase;
 use tests\unit\Util\TestLoggingUtil;
+use Magento\FunctionalTestingFramework\Util\GenerationErrorHandler;
 
 class AnnotationExtractorTest extends TestCase
 {
@@ -195,14 +196,20 @@ public function testTestCaseIdUniqueness()
         $extractor = new AnnotationExtractor();
         $extractor->extractAnnotations($firstTestAnnotation, "firstTest");
         $extractor->extractAnnotations($secondTestannotation, "secondTest");
+        $extractor->validateTestCaseIdTitleUniqueness();
 
-        //Expect Exception
-        $this->expectException(\Magento\FunctionalTestingFramework\Exceptions\XmlException::class);
-        $this->expectExceptionMessage("TestCaseId and Title pairs must be unique:\n\n" .
-            "TestCaseId: 'MQE-0001' Title: 'TEST TITLE' in Tests 'firstTest', 'secondTest'");
+        // assert that no exception for validateTestCaseIdTitleUniqueness
+        // and validation error is stored in GenerationErrorHandler
+        $errorMessage = '/' . preg_quote("TestCaseId and Title pairs is not unique in Tests 'firstTest', 'secondTest'") . '/';
+        TestLoggingUtil::getInstance()->validateMockLogStatmentRegex('error', $errorMessage, []);
+        $testErrors = GenerationErrorHandler::getInstance()->getErrorsByType('test');
+        $this->assertArrayHasKey('firstTest', $testErrors);
+        $this->assertArrayHasKey('secondTest', $testErrors);
+    }
 
-        //Trigger Exception
-        $extractor->validateTestCaseIdTitleUniqueness();
+    public function tearDown(): void
+    {
+        GenerationErrorHandler::getInstance()->reset();
     }
 
     /**