From a1873d70552145028f4160d9ff44c9de4378132d Mon Sep 17 00:00:00 2001 From: jrfnl Date: Tue, 18 Apr 2023 22:29:08 +0200 Subject: [PATCH] Reports: add end-to-end tests ... for quite a few report types. The Reports which PHPCS can generate, were until now not covered by tests and therefore had no safeguards/QA whatsoever. This commit sets up an initial end-to-end test suite with tests for common report types. It doesn't have an opinion on whether or not the reports as currently generated are formatted correctly. For now, it just codifies the current functionality. For lack of documentation about this format in the PHPUnit docs, the test docs for PHP Core are the closest available to explain the available sections and how to apply these: https://qa.php.net/phpt_details.php With this initial setup done, it should be fairly straight-forward to expand this end-to-end test suite to cover more report types and to add tests for other command-line options as well. Mind: these type of tests are generally slow, so unit tests/integration tests should be preferred. --- package.xml | 62 +++++++++++++++ phpcs.xml.dist | 1 + phpunit.xml.dist | 4 + .../ValidatePEAR/ValidatePEARPackageXML.php | 8 +- .../EndToEnd/Fixtures/Reports/CleanClass.php | 12 +++ tests/EndToEnd/Fixtures/Reports/Dirty.php | 14 ++++ tests/EndToEnd/Reports/checkstyle.phpt | 31 ++++++++ .../EndToEnd/Reports/code-no-errorcodes.phpt | 73 +++++++++++++++++ .../Reports/code-with-errorcodes.phpt | 79 +++++++++++++++++++ tests/EndToEnd/Reports/csv.phpt | 27 +++++++ .../EndToEnd/Reports/emacs-no-errorcodes.phpt | 26 ++++++ .../Reports/emacs-with-errorcodes.phpt | 26 ++++++ .../EndToEnd/Reports/full-no-errorcodes.phpt | 42 ++++++++++ .../Reports/full-with-errorcodes.phpt | 48 +++++++++++ .../Reports/gitblame-no-errorcodes.phpt | 29 +++++++ .../Reports/gitblame-with-errorcodes.phpt | 35 ++++++++ tests/EndToEnd/Reports/json.phpt | 19 +++++ tests/EndToEnd/Reports/junit.phpt | 50 ++++++++++++ .../Reports/source-no-errorcodes.phpt | 34 ++++++++ .../Reports/source-with-errorcodes.phpt | 34 ++++++++ tests/EndToEnd/Reports/summary.phpt | 29 +++++++ tests/EndToEnd/Reports/xml.phpt | 31 ++++++++ 22 files changed, 710 insertions(+), 4 deletions(-) create mode 100644 tests/EndToEnd/Fixtures/Reports/CleanClass.php create mode 100644 tests/EndToEnd/Fixtures/Reports/Dirty.php create mode 100644 tests/EndToEnd/Reports/checkstyle.phpt create mode 100644 tests/EndToEnd/Reports/code-no-errorcodes.phpt create mode 100644 tests/EndToEnd/Reports/code-with-errorcodes.phpt create mode 100644 tests/EndToEnd/Reports/csv.phpt create mode 100644 tests/EndToEnd/Reports/emacs-no-errorcodes.phpt create mode 100644 tests/EndToEnd/Reports/emacs-with-errorcodes.phpt create mode 100644 tests/EndToEnd/Reports/full-no-errorcodes.phpt create mode 100644 tests/EndToEnd/Reports/full-with-errorcodes.phpt create mode 100644 tests/EndToEnd/Reports/gitblame-no-errorcodes.phpt create mode 100644 tests/EndToEnd/Reports/gitblame-with-errorcodes.phpt create mode 100644 tests/EndToEnd/Reports/json.phpt create mode 100644 tests/EndToEnd/Reports/junit.phpt create mode 100644 tests/EndToEnd/Reports/source-no-errorcodes.phpt create mode 100644 tests/EndToEnd/Reports/source-with-errorcodes.phpt create mode 100644 tests/EndToEnd/Reports/summary.phpt create mode 100644 tests/EndToEnd/Reports/xml.phpt diff --git a/package.xml b/package.xml index a5c8053d05..d4b51b910b 100644 --- a/package.xml +++ b/package.xml @@ -273,6 +273,32 @@ http://pear.php.net/dtd/package-2.0.xsd"> + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -2264,6 +2290,24 @@ http://pear.php.net/dtd/package-2.0.xsd"> + + + + + + + + + + + + + + + + + + @@ -2388,6 +2432,24 @@ http://pear.php.net/dtd/package-2.0.xsd"> + + + + + + + + + + + + + + + + + + diff --git a/phpcs.xml.dist b/phpcs.xml.dist index 3c54610094..a0ee6e2073 100644 --- a/phpcs.xml.dist +++ b/phpcs.xml.dist @@ -11,6 +11,7 @@ */src/Standards/*/Tests/*\.(inc|css|js)$ */tests/Core/*/*\.(inc|css|js)$ */tests/Core/*/Fixtures/*\.php$ + */tests/EndToEnd/Fixtures/* diff --git a/phpunit.xml.dist b/phpunit.xml.dist index 34b4afcded..d05dbb6b35 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -4,5 +4,9 @@ tests/AllTests.php + + tests/EndToEnd/ + tests/EndToEnd/Fixtures + diff --git a/scripts/ValidatePEAR/ValidatePEARPackageXML.php b/scripts/ValidatePEAR/ValidatePEARPackageXML.php index 5c1d2d1f0f..981eda675b 100644 --- a/scripts/ValidatePEAR/ValidatePEARPackageXML.php +++ b/scripts/ValidatePEAR/ValidatePEARPackageXML.php @@ -131,12 +131,12 @@ protected function checkContents() $srcFiles = (new FileList( $this->projectRoot.'src/', $this->projectRoot, - '`\.(css|fixed|inc|js|php|xml)$`Di' + '`\.(css|fixed|inc|js|phpt|php|xml)$`Di' ))->getList(); $testsFiles = (new FileList( $this->projectRoot.'tests/', $this->projectRoot, - '`\.(css|inc|js|php|xml)$`Di' + '`\.(css|inc|js|phpt|php|xml)$`Di' ))->getList(); $files = array_merge($srcFiles, $testsFiles); @@ -317,7 +317,7 @@ protected function checkPHPRelease() } // Check validity of the tags for files in the tests root subdirectories. - if (preg_match('`^tests/.+\.(php|inc|js|css|xml)$`', $name) === 1 + if (preg_match('`^tests/.+\.(phpt|php|inc|js|css|xml)$`', $name) === 1 && $as === str_replace('tests/', 'CodeSniffer/', $name) ) { continue; @@ -338,7 +338,7 @@ protected function checkPHPRelease() * Verify that all files in the `tests` directory are listed in both `` tags. */ - $testFiles = (new FileList($this->projectRoot.'tests/', $this->projectRoot, '`\.(inc|php|js|css|xml)$`Di'))->getList(); + $testFiles = (new FileList($this->projectRoot.'tests/', $this->projectRoot, '`\.(inc|phpt|php|js|css|xml)$`Di'))->getList(); foreach ($testFiles as $file) { foreach ($listedFiles as $key => $listed) { diff --git a/tests/EndToEnd/Fixtures/Reports/CleanClass.php b/tests/EndToEnd/Fixtures/Reports/CleanClass.php new file mode 100644 index 0000000000..33b586afdd --- /dev/null +++ b/tests/EndToEnd/Fixtures/Reports/CleanClass.php @@ -0,0 +1,12 @@ + +--ARGS-- +./tests/EndToEnd/Fixtures/Reports/ -q --no-colors --report-width=80 --basepath=./tests/EndToEnd/Fixtures/Reports/ --standard=PSR1 --report=Checkstyle + +--FILE-- + + + + + + + + + + + + + diff --git a/tests/EndToEnd/Reports/code-no-errorcodes.phpt b/tests/EndToEnd/Reports/code-no-errorcodes.phpt new file mode 100644 index 0000000000..d00ce9f344 --- /dev/null +++ b/tests/EndToEnd/Reports/code-no-errorcodes.phpt @@ -0,0 +1,73 @@ +--TEST-- +Report: Code, no error codes + +--SKIPIF-- + +--ARGS-- +./tests/EndToEnd/Fixtures/Reports/ -q --no-colors --report-width=80 --basepath=./tests/EndToEnd/Fixtures/Reports/ --standard=PSR1 --report=Code + +--FILE-- +> 1: > 6: class dirty_class { + 7: const lowerCase = false; + 8: +-------------------------------------------------------------------------------- +LINE 7: ERROR Class constants must be uppercase; expected LOWERCASE but found + lowerCase +-------------------------------------------------------------------------------- + 5:%w + 6: class dirty_class { +>> 7: const lowerCase = false; + 8:%w + 9: public function My_Method() {} +-------------------------------------------------------------------------------- +LINE 9: ERROR Method name "dirty_class::My_Method" is not in camel caps + format +-------------------------------------------------------------------------------- + 7: const lowerCase = false; + 8:%w +>> 9: public function My_Method() {} + 10: } + 11: +-------------------------------------------------------------------------------- +LINE 12: ERROR Each class must be in a file by itself +LINE 12: ERROR Each class must be in a namespace of at least one level (a + top-level vendor name) +LINE 12: ERROR Class name "Second_class" is not in PascalCase format +-------------------------------------------------------------------------------- + 10: } + 11:%w +>> 12: class Second_class {} + 13:%w + 14: $obj = new dirty_class(); +-------------------------------------------------------------------------------- +Time: %f secs; Memory: %dMB diff --git a/tests/EndToEnd/Reports/code-with-errorcodes.phpt b/tests/EndToEnd/Reports/code-with-errorcodes.phpt new file mode 100644 index 0000000000..dac17636d3 --- /dev/null +++ b/tests/EndToEnd/Reports/code-with-errorcodes.phpt @@ -0,0 +1,79 @@ +--TEST-- +Report: Code, with error codes + +--SKIPIF-- + +--ARGS-- +./tests/EndToEnd/Fixtures/Reports/ -qs --no-colors --report-width=80 --basepath=./tests/EndToEnd/Fixtures/Reports/ --standard=PSR1 --report=Code + +--FILE-- +> 1: > 6: class dirty_class { + 7: const lowerCase = false; + 8: +-------------------------------------------------------------------------------- +LINE 7: ERROR Class constants must be uppercase; expected LOWERCASE but found + lowerCase + (Generic.NamingConventions.UpperCaseConstantName.ClassConstantNotUpperCase) +-------------------------------------------------------------------------------- + 5:%w + 6: class dirty_class { +>> 7: const lowerCase = false; + 8:%w + 9: public function My_Method() {} +-------------------------------------------------------------------------------- +LINE 9: ERROR Method name "dirty_class::My_Method" is not in camel caps + format (PSR1.Methods.CamelCapsMethodName.NotCamelCaps) +-------------------------------------------------------------------------------- + 7: const lowerCase = false; + 8:%w +>> 9: public function My_Method() {} + 10: } + 11: +-------------------------------------------------------------------------------- +LINE 12: ERROR Each class must be in a file by itself + (PSR1.Classes.ClassDeclaration.MultipleClasses) +LINE 12: ERROR Each class must be in a namespace of at least one level (a + top-level vendor name) + (PSR1.Classes.ClassDeclaration.MissingNamespace) +LINE 12: ERROR Class name "Second_class" is not in PascalCase format + (Squiz.Classes.ValidClassName.NotCamelCaps) +-------------------------------------------------------------------------------- + 10: } + 11:%w +>> 12: class Second_class {} + 13:%w + 14: $obj = new dirty_class(); +-------------------------------------------------------------------------------- +Time: %f secs; Memory: %dMB diff --git a/tests/EndToEnd/Reports/csv.phpt b/tests/EndToEnd/Reports/csv.phpt new file mode 100644 index 0000000000..d87c1a3685 --- /dev/null +++ b/tests/EndToEnd/Reports/csv.phpt @@ -0,0 +1,27 @@ +--TEST-- +Report: Csv + +--SKIPIF-- + +--ARGS-- +./tests/EndToEnd/Fixtures/Reports/ -q --no-colors --report-width=80 --basepath=./tests/EndToEnd/Fixtures/Reports/ --standard=PSR1 --report=Csv + +--FILE-- + +--ARGS-- +./tests/EndToEnd/Fixtures/Reports/ -q --no-colors --report-width=80 --basepath=./tests/EndToEnd/Fixtures/Reports/ --standard=PSR1 --report=Emacs + +--FILE-- + +--ARGS-- +./tests/EndToEnd/Fixtures/Reports/ -qs --no-colors --report-width=80 --basepath=./tests/EndToEnd/Fixtures/Reports/ --standard=PSR1 --report=Emacs + +--FILE-- + +--ARGS-- +./tests/EndToEnd/Fixtures/Reports/ -q --no-colors --report-width=80 --basepath=./tests/EndToEnd/Fixtures/Reports/ --standard=PSR1 --report=Full + +--FILE-- + +--ARGS-- +./tests/EndToEnd/Fixtures/Reports/ -qs --no-colors --report-width=80 --basepath=./tests/EndToEnd/Fixtures/Reports/ --standard=PSR1 --report=Full + +--FILE-- + +--ARGS-- +./tests/EndToEnd/Fixtures/Reports/ -q --no-colors --report-width=80 --basepath=./tests/EndToEnd/Fixtures/Reports/ --standard=PSR1 --report=Gitblame + +--FILE-- + +--ARGS-- +./tests/EndToEnd/Fixtures/Reports/ -qs --no-colors --report-width=80 --basepath=./tests/EndToEnd/Fixtures/Reports/ --standard=PSR1 --report=Gitblame + +--FILE-- + +--ARGS-- +./tests/EndToEnd/Fixtures/Reports/ -q --no-colors --report-width=80 --basepath=./tests/EndToEnd/Fixtures/Reports/ --standard=PSR1 --report=Json + +--FILE-- + +--ARGS-- +./tests/EndToEnd/Fixtures/Reports/ -q --no-colors --report-width=80 --basepath=./tests/EndToEnd/Fixtures/Reports/ --standard=PSR1 --report=Junit + +--FILE-- + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/EndToEnd/Reports/source-no-errorcodes.phpt b/tests/EndToEnd/Reports/source-no-errorcodes.phpt new file mode 100644 index 0000000000..f5518ff002 --- /dev/null +++ b/tests/EndToEnd/Reports/source-no-errorcodes.phpt @@ -0,0 +1,34 @@ +--TEST-- +Report: Source, no error codes + +--SKIPIF-- + +--ARGS-- +./tests/EndToEnd/Fixtures/Reports/ -q --no-colors --report-width=80 --basepath=./tests/EndToEnd/Fixtures/Reports/ --standard=PSR1 --report=Source + +--FILE-- + +--ARGS-- +./tests/EndToEnd/Fixtures/Reports/ -qs --no-colors --report-width=80 --basepath=./tests/EndToEnd/Fixtures/Reports/ --standard=PSR1 --report=Source + +--FILE-- + +--ARGS-- +./tests/EndToEnd/Fixtures/Reports/ -q --no-colors --report-width=80 --basepath=./tests/EndToEnd/Fixtures/Reports/ --standard=PSR1 --report=Summary + +--FILE-- + +--ARGS-- +./tests/EndToEnd/Fixtures/Reports/ -q --no-colors --report-width=80 --basepath=./tests/EndToEnd/Fixtures/Reports/ --standard=PSR1 --report=Xml + +--FILE-- + + + + A file should declare new symbols (classes, functions, constants, etc.) and cause no other side effects, or it should execute logic with side effects, but should not do both. The first symbol is defined on line 6 and the first side effect is on line 14. + Each class must be in a namespace of at least one level (a top-level vendor name) + Class name "dirty_class" is not in PascalCase format + Class constants must be uppercase; expected LOWERCASE but found lowerCase + Method name "dirty_class::My_Method" is not in camel caps format + Each class must be in a file by itself + Each class must be in a namespace of at least one level (a top-level vendor name) + Class name "Second_class" is not in PascalCase format + +