Merge pull request #245 from wp-cli/fix/missing-coverage-files

Create multiple coverage files and merge them all together

Author: swissspidy

src/Context/FeatureContext.php

@@ -9,7 +9,14 @@
 use Behat\Testwork\Hook\Scope\BeforeSuiteScope;
 use Behat\Behat\Hook\Scope\AfterFeatureScope;
 use Behat\Behat\Hook\Scope\BeforeFeatureScope;
+use Behat\Behat\Hook\Scope\BeforeStepScope;
+use SebastianBergmann\CodeCoverage\Report\Clover;
+use SebastianBergmann\CodeCoverage\Driver\Selector;
+use SebastianBergmann\CodeCoverage\Filter;
+use SebastianBergmann\CodeCoverage\CodeCoverage;
+use SebastianBergmann\Environment\Runtime;
 use RuntimeException;
+use DirectoryIterator;
 use WP_CLI\Process;
 use WP_CLI\Utils;
 use WP_CLI\WpOrgApi;
@@ -131,6 +138,13 @@ class FeatureContext implements SnippetAcceptingContext {
 	 */
 	private $scenario;
 
+	/**
+	 * Line of the current step.
+	 *
+	 * @var int
+	 */
+	private $step_line = 0;
+
 	/**
 	 * @BeforeFeature
 	 */
@@ -145,11 +159,19 @@ public function store_scenario( BeforeScenarioScope $scope ) {
 		$this->scenario = $scope->getScenario();
 	}
 
+	/**
+	 * @BeforeStep
+	 */
+	public function store_step( BeforeStepScope $scope ) {
+		$this->step_line = $scope->getStep()->getLine();
+	}
+
 	/**
 	 * @AfterScenario
 	 */
 	public function forget_scenario( AfterScenarioScope $scope ) {
-		$this->scenario = null;
+		$this->step_line = 0;
+		$this->scenario  = null;
 	}
 
 	/**
@@ -159,6 +181,34 @@ public static function forget_feature( AfterFeatureScope $scope ) {
 		self::$feature = null;
 	}
 
+	/**
+	 * @AfterSuite
+	 */
+	public static function merge_coverage_reports() {
+		$with_code_coverage = (string) getenv( 'WP_CLI_TEST_COVERAGE' );
+
+		if ( ! \in_array( $with_code_coverage, [ 'true', '1' ], true ) ) {
+			return;
+		}
+
+		$filter   = new Filter();
+		$coverage = new CodeCoverage(
+			( new Selector() )->forLineCoverage( $filter ),
+			$filter
+		);
+
+		foreach ( new DirectoryIterator( self::$behat_run_dir . '/build/logs' ) as $file ) {
+			if ( ! $file->isFile() || 'cov' !== $file->getExtension() ) {
+				continue;
+			}
+
+			$coverage->merge( include $file->getPathname() );
+			unlink( $file->getPathname() );
+		}
+
+		( new Clover() )->process( $coverage, self::$behat_run_dir . '/build/logs/behat-coverage.xml' );
+	}
+
 	/**
 	 * Get the path to the Composer vendor folder.
 	 *
@@ -298,7 +348,14 @@ private static function get_process_env_variables() {
 		];
 
 		$with_code_coverage = (string) getenv( 'WP_CLI_TEST_COVERAGE' );
+
 		if ( \in_array( $with_code_coverage, [ 'true', '1' ], true ) ) {
+			$has_coverage_driver = ( new Runtime() )->hasXdebug() || ( new Runtime() )->hasPCOV();
+
+			if ( ! $has_coverage_driver ) {
+				throw new RuntimeException( 'No coverage driver available. Re-run script with `--xdebug` flag, i.e. `composer behat -- --xdebug`.' );
+			}
+
 			$coverage_require_file = self::$behat_run_dir . '/vendor/wp-cli/wp-cli-tests/utils/generate-coverage.php';
 			if ( ! file_exists( $coverage_require_file ) ) {
 				// This file is not vendored inside the wp-cli-tests project
@@ -871,8 +928,8 @@ public function download_phar( $version = 'same' ) {
 		);
 
 		$this->variables['PHAR_PATH'] = $this->variables['RUN_DIR'] . '/'
-										. uniqid( 'wp-cli-download-', true )
-										. '.phar';
+			. uniqid( 'wp-cli-download-', true )
+			. '.phar';
 
 		Process::create(
 			Utils\esc_cmd(
@@ -957,6 +1014,8 @@ public function proc( $command, $assoc_args = [], $path = '' ) {
 			$env['BEHAT_SCENARIO_TITLE'] = $this->scenario->getTitle();
 		}
 
+		$env['BEHAT_STEP_LINE'] = $this->step_line;
+
 		$env['WP_CLI_TEST_DBTYPE'] = self::$db_type;
 
 		if ( isset( $this->variables['RUN_DIR'] ) ) {

utils/generate-coverage.php

@@ -1,16 +1,32 @@
 <?php
 
 /**
- * This script is added via `--require` to the WP-CLI commands executed by the Behat test runner.
+ * This script is added via the `WP_CLI_REQUIRE` environment variable to the WP-CLI commands executed by the Behat test runner.
  * It starts coverage collection right away and registers a shutdown hook to complete it
  * after the respective WP-CLI command has finished.
  */
 
 use SebastianBergmann\CodeCoverage\CodeCoverage;
 use SebastianBergmann\CodeCoverage\Driver\Selector;
 use SebastianBergmann\CodeCoverage\Filter;
-use SebastianBergmann\CodeCoverage\Report\Clover;
+use SebastianBergmann\CodeCoverage\Report\PHP as PHPReport;
 
+/*
+ * The names of the current feature and scenario are passed on from the Behat test runner
+ * to this script through environment variables.
+ */
+$feature   = getenv( 'BEHAT_FEATURE_TITLE' );
+$scenario  = getenv( 'BEHAT_SCENARIO_TITLE' );
+$step_line = (int) getenv( 'BEHAT_STEP_LINE' );
+$name      = "{$feature} - {$scenario} - {$step_line}";
+
+/*
+ * Do not run coverage if they are empty, which means we are running some command
+ * during test preparation, e.g. the `wp core download` in `FeatureContext::prepare()`.
+ */
+if ( empty( $feature ) | empty( $scenario ) ) {
+	return;
+}
 
 $project_dir = (string) getenv( 'TEST_RUN_DIR' );
 
@@ -65,30 +81,22 @@ function ( $file ) {
 	$filter
 );
 
-/*
- * The names of the current feature and scenario are passed on from the Behat test runner
- * to this script through environment variables `BEHAT_FEATURE_TITLE` & `BEHAT_SCENARIO_TITLE`.
- */
-$feature  = getenv( 'BEHAT_FEATURE_TITLE' );
-$scenario = getenv( 'BEHAT_SCENARIO_TITLE' );
-$name     = "{$feature} - {$scenario}";
-
 $coverage->start( $name );
 
 register_shutdown_function(
-	static function () use ( $coverage, $feature, $scenario, $name, $project_dir ) {
+	static function () use ( $coverage, $feature, $scenario, $step_line, $name, $project_dir ) {
 		$coverage->stop();
 
 		$feature_suffix  = preg_replace( '/[^a-z0-9]+/', '-', strtolower( $feature ) );
 		$scenario_suffix = preg_replace( '/[^a-z0-9]+/', '-', strtolower( $scenario ) );
 		$db_type         = strtolower( getenv( 'WP_CLI_TEST_DBTYPE' ) );
-		$destination     = "$project_dir/build/logs/$feature_suffix-$scenario_suffix-$db_type.xml";
+		$destination     = "$project_dir/build/logs/$feature_suffix-$scenario_suffix-$step_line-$db_type.cov";
 
 		$dir = dirname( $destination );
 		if ( ! file_exists( $dir ) ) {
 			mkdir( $dir, 0777, true /*recursive*/ );
 		}
 
-		( new Clover() )->process( $coverage, $destination, $name );
+		( new PHPReport() )->process( $coverage, $destination );
 	}
 );