Merge pull request PHPCSStandards#144 from PHPCSStandards/feature/unit-tests-add-covers-annotations

CI/QA: start recording code coverage

Author: jrfnl

.github/CONTRIBUTING.md

@@ -152,9 +152,8 @@ To help you with this, a number of convenience scripts are available:
 * `composer cs` will check for code style violations.
 * `composer cbf` will run the autofixers for code style violations.
 * `composer test` will run the unit tests.
-* `composer coverage` will run the unit tests with code coverage.
-    Note: you may want to use a custom `phpunit.xml` overload config file to tell PHPUnit where to place an HTML report.
-    Alternative run it like so: `composer coverage -- --coverage-html /path/to/report-dir/` to specify the location for the HTML report on the command line.
+* `composer coverage` will run the unit tests with code coverage and show a text summary.
+* `composer coverage-local` will run the unit tests with code coverage and generate an HTML coverage report, which will be placed in a `build/coverage-html` subdirectory.
 * `composer build` will build the phpcs.phar and phpcbf.phar files.
 
 N.B.: You can ignore any skipped tests as these are for external tools.

.github/workflows/quicktest.yml

@@ -54,7 +54,7 @@ jobs:
         run: php bin/phpcs --config-set php_path php
 
       - name: 'PHPUnit: run the tests'
-        run: vendor/bin/phpunit tests/AllTests.php
+        run: vendor/bin/phpunit tests/AllTests.php --no-coverage
 
       # Note: The code style check is run as an integration test.
       - name: 'PHPCS: check code style without cache, no parallel'

.github/workflows/test.yml

@@ -74,7 +74,14 @@ jobs:
         custom_ini: [false]
 
         include:
-          # Builds running the basic tests with different PHP ini settings.
+          # Skip test runs on builds which are also run for in the coverage job.
+          # Note: the tests on PHP 7.2 will still be run as the coverage build is uses custom_ini for that version.
+          - php: '5.4'
+            skip_tests: true
+          - php: '8.3'
+            skip_tests: true
+
+          # Extra builds running only the unit tests with different PHP ini settings.
           - php: '5.5'
             custom_ini: true
           - php: '7.0'
@@ -137,8 +144,9 @@ jobs:
       - name: 'PHPCS: set the path to PHP'
         run: php bin/phpcs --config-set php_path php
 
-      - name: 'PHPUnit: run the tests'
-        run: vendor/bin/phpunit tests/AllTests.php
+      - name: 'PHPUnit: run the tests without code coverage'
+        if: ${{ matrix.skip_tests != true }}
+        run: vendor/bin/phpunit tests/AllTests.php --no-coverage
 
       - name: 'PHPCS: check code style without cache, no parallel'
         if: ${{ matrix.custom_ini == false && matrix.php != '7.4' }}
@@ -163,3 +171,92 @@ jobs:
       - name: 'PHPCS: check code style using the Phar file'
         if: ${{ matrix.custom_ini == false }}
         run: php phpcs.phar
+
+  coverage:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        include:
+          - php: '5.4'
+            custom_ini: false
+          - php: '7.2'
+            custom_ini: true
+          - php: '8.3'
+            custom_ini: false
+
+    name: "Coverage: ${{ matrix.php }} ${{ matrix.custom_ini && ' with custom ini settings' || '' }}"
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup ini config
+        id: set_ini
+        run: |
+          # Set the "short_open_tag" ini to make sure specific conditions are tested.
+          # Also turn on error_reporting to ensure all notices are shown.
+          if [[ ${{ matrix.custom_ini }} == true && "${{ startsWith( matrix.php, '5.' ) }}" == true ]]; then
+            echo 'PHP_INI=error_reporting=-1, display_errors=On, date.timezone=Australia/Sydney, short_open_tag=On, asp_tags=On' >> $GITHUB_OUTPUT
+          elif [[ ${{ matrix.custom_ini }} == true && "${{ startsWith( matrix.php, '7.' ) }}" == true ]]; then
+            echo 'PHP_INI=error_reporting=-1, display_errors=On, date.timezone=Australia/Sydney, short_open_tag=On' >> $GITHUB_OUTPUT
+          else
+            echo 'PHP_INI=error_reporting=-1, display_errors=On' >> $GITHUB_OUTPUT
+          fi
+
+      - name: Install PHP
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: ${{ matrix.php }}
+          ini-values: ${{ steps.set_ini.outputs.PHP_INI }}
+          coverage: xdebug
+
+      # This action also handles the caching of the dependencies.
+      - name: Set up node
+        if: ${{ matrix.custom_ini == false }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Install external tools used in tests
+        if: ${{ matrix.custom_ini == false }}
+        run: >
+          npm install -g --fund false
+          csslint
+          eslint
+          jshint
+
+      # Install dependencies and handle caching in one go.
+      # @link https://github.com/marketplace/actions/install-php-dependencies-with-composer
+      - name: Install Composer dependencies
+        uses: "ramsey/composer-install@v2"
+        with:
+          # Bust the cache at least once a month - output format: YYYY-MM.
+          custom-cache-suffix: $(date -u "+%Y-%m")
+
+      - name: 'PHPCS: set the path to PHP'
+        run: php bin/phpcs --config-set php_path php
+
+      - name: 'PHPUnit: run the tests with code coverage'
+        run: vendor/bin/phpunit tests/AllTests.php
+
+      - name: Upload coverage results to Coveralls
+        if: ${{ success() }}
+        uses: coverallsapp/github-action@v2
+        with:
+          format: clover
+          file: build/logs/clover.xml
+          flag-name: php-${{ matrix.php }}-custom-ini-${{ matrix.custom_ini }}
+          parallel: true
+
+  coveralls-finish:
+    needs: coverage
+    if: always() && needs.coverage.result == 'success'
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Coveralls Finished
+        uses: coverallsapp/github-action@v2
+        with:
+          parallel-finished: true

.gitignore

@@ -2,6 +2,7 @@
 /phpcs.xml
 /phpunit.xml
 .phpunit.result.cache
+/build/
 .idea/*
 /vendor/
 composer.lock

README.md

@@ -6,6 +6,7 @@ PHP_CodeSniffer
 [![Latest Stable Version](http://poser.pugx.org/phpcsstandards/php_codesniffer/v)](https://github.com/PHPCSStandards/PHP_CodeSniffer/releases)
 [![Validate](https://github.com/PHPCSStandards/PHP_CodeSniffer/actions/workflows/validate.yml/badge.svg?branch=master)](https://github.com/PHPCSStandards/PHP_CodeSniffer/actions/workflows/validate.yml)
 [![Test](https://github.com/PHPCSStandards/PHP_CodeSniffer/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/PHPCSStandards/PHP_CodeSniffer/actions/workflows/test.yml)
+[![Coverage Status](https://coveralls.io/repos/github/PHPCSStandards/PHP_CodeSniffer/badge.svg?branch=master)](https://coveralls.io/github/PHPCSStandards/PHP_CodeSniffer?branch=master)
 [![License](http://poser.pugx.org/phpcsstandards/php_codesniffer/license)](https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt)
 
 ![Minimum PHP Version](https://img.shields.io/packagist/php-v/squizlabs/php_codesniffer.svg?maxAge=3600)

composer.json

@@ -62,6 +62,10 @@
             "Composer\\Config::disableProcessTimeout",
             "@php ./vendor/phpunit/phpunit/phpunit tests/AllTests.php -d max_execution_time=0"
         ],
+        "coverage-local": [
+            "Composer\\Config::disableProcessTimeout",
+            "@php ./vendor/phpunit/phpunit/phpunit tests/AllTests.php --coverage-html ./build/coverage-html -d max_execution_time=0"
+        ],
         "build": [
             "Composer\\Config::disableProcessTimeout",
             "@php -d phar.readonly=0 -f ./scripts/build-phar.php"
@@ -76,6 +80,7 @@
         "cbf": "Fix code style violations.",
         "test": "Run the unit tests without code coverage.",
         "coverage": "Run the unit tests with code coverage.",
+        "coverage-local": "Run the unit tests with code coverage and generate an HTML report in a 'build' directory.",
         "build": "Create PHAR files for PHPCS and PHPCBF.",
         "check-all": "Run all checks (phpcs, tests)."
     }

phpunit.xml.dist

@@ -9,10 +9,26 @@
     convertWarningsToExceptions="true"
     convertNoticesToExceptions="true"
     convertDeprecationsToExceptions="true"
+    forceCoversAnnotation="true"
     >
     <testsuites>
         <testsuite name="PHP_CodeSniffer Test Suite">
             <file>tests/AllTests.php</file>
         </testsuite>
     </testsuites>
+
+    <filter>
+        <whitelist addUncoveredFilesFromWhitelist="true" processUncoveredFilesFromWhitelist="false">
+            <directory suffix=".php">./src</directory>
+            <file>./autoload.php</file>
+            <exclude>
+                <directory suffix="UnitTest.php">./src/Standards</directory>
+            </exclude>
+        </whitelist>
+    </filter>
+
+    <logging>
+        <log type="coverage-text" target="php://stdout" showOnlySummary="true"/>
+        <log type="coverage-clover" target="build/logs/clover.xml"/>
+    </logging>
 </phpunit>

src/Standards/Generic/Tests/Arrays/ArrayIndentUnitTest.php

@@ -11,6 +11,11 @@
 
 use PHP_CodeSniffer\Tests\Standards\AbstractSniffUnitTest;
 
+/**
+ * Unit test class for the ArrayIndent sniff.
+ *
+ * @covers \PHP_CodeSniffer\Standards\Generic\Sniffs\Arrays\ArrayIndentSniff
+ */
 class ArrayIndentUnitTest extends AbstractSniffUnitTest
 {
 

src/Standards/Generic/Tests/Arrays/DisallowLongArraySyntaxUnitTest.php

@@ -11,6 +11,11 @@
 
 use PHP_CodeSniffer\Tests\Standards\AbstractSniffUnitTest;
 
+/**
+ * Unit test class for the DisallowLongArraySyntax sniff.
+ *
+ * @covers \PHP_CodeSniffer\Standards\Generic\Sniffs\Arrays\DisallowLongArraySyntaxSniff
+ */
 class DisallowLongArraySyntaxUnitTest extends AbstractSniffUnitTest
 {
 

src/Standards/Generic/Tests/Arrays/DisallowShortArraySyntaxUnitTest.php

@@ -11,6 +11,11 @@
 
 use PHP_CodeSniffer\Tests\Standards\AbstractSniffUnitTest;
 
+/**
+ * Unit test class for the DisallowShortArraySyntax sniff.
+ *
+ * @covers \PHP_CodeSniffer\Standards\Generic\Sniffs\Arrays\DisallowShortArraySyntaxSniff
+ */
 class DisallowShortArraySyntaxUnitTest extends AbstractSniffUnitTest
 {