Merge pull request #16 from wp-cli/13-rerun-failed-scenarios

Explain how to re-run failed scenarios

Author: schlessera

.travis.yml

@@ -2,6 +2,7 @@ sudo: false
 dist: trusty
 
 language: php
+php: 7.2
 
 notifications:
   email:
@@ -21,23 +22,6 @@ env:
     - PATH="$TRAVIS_BUILD_DIR/vendor/bin:$PATH"
     - WP_CLI_BIN_DIR="$TRAVIS_BUILD_DIR/vendor/bin"
 
-matrix:
-  include:
-    - php: 7.2
-      env: WP_VERSION=latest
-    - php: 7.1
-      env: WP_VERSION=latest
-    - php: 7.0
-      env: WP_VERSION=latest
-    - php: 5.6
-      env: WP_VERSION=latest
-    - php: 5.6
-      env: WP_VERSION=3.7.11
-    - php: 5.6
-      env: WP_VERSION=trunk
-    - php: 5.4
-      env: WP_VERSION=latest
-
 before_install:
   - |
     # Remove Xdebug for a huge performance increase:
@@ -46,13 +30,45 @@ before_install:
     else
       echo "xdebug.ini does not exist"
     fi
+  - |
+    # Raise PHP memory limit to 2048MB
+    echo 'memory_limit = 2048M' >> ~/.phpenv/versions/$(phpenv version-name)/etc/conf.d/travis.ini
+  - composer validate
 
 install:
   - composer install
   - composer prepare-tests
 
-before_script:
-  - composer validate
-
 script:
-  - composer test
+  - composer phpunit
+  - composer behat || composer behat-rerun
+
+jobs:
+  include:
+    - stage: sniff
+      script:
+        - composer lint
+        - composer phpcs
+      env: BUILD=sniff
+    - stage: test
+      php: 7.2
+      env: WP_VERSION=latest
+    - stage: test
+      php: 7.1
+      env: WP_VERSION=latest
+    - stage: test
+      php: 7.0
+      env: WP_VERSION=latest
+    - stage: test
+      php: 5.6
+      env: WP_VERSION=latest
+    - stage: test
+      php: 5.6
+      env: WP_VERSION=3.7.11
+    - stage: test
+      php: 5.6
+      env: WP_VERSION=trunk
+    - stage: test
+      php: 5.4
+      dist: precise
+      env: WP_VERSION=latest

README.md

@@ -52,7 +52,7 @@ To make use of the WP-CLI testing framework, you need to complete the following
 	```
 	This will make sure that the automated Behat system works across all platforms. This is needed on Windows.
 
-4. Update your composer dependencies and regenerate your autoloader and binary folders:
+5. Update your composer dependencies and regenerate your autoloader and binary folders:
 	```bash
 	composer update
 	```
@@ -81,6 +81,72 @@ Prepending with the double dash is needed because the arguments would otherwise
 
 ### Controlling the test environment
 
+#### WordPress Version
+
+You can run the tests against a specific version of WordPress by setting the `WP_VERSION` environment variable.
+
+This variable understands any numeric version, as well as the special terms `latest` and `trunk`.
+
+Note: This only applies to the Behat functional tests. All other tests never load WordPress.
+
+Here's how to run your tests against the latest trunk version of WordPress:
+```bash
+WP_VERSION=trunk composer behat
+```
+
+#### WP-CLI Binary
+
+You can run the tests against a specific WP-CLI binary, instead of using the one that has been built in your project's `vendor/bin` folder.
+
+This can be useful to run your tests against a specific Phar version of WP_CLI.
+
+To do this, you can set the `WP_CLI_BIN_DIR` environment variable to point to a folder that contains an executable `wp` binary. Note: the binary has to be named `wp` to be properly recognized.
+
+As an example, here's how to run your tests against a specific Phar version you've downloaded.
+```bash
+# Prepare the binary you've downloaded into the ~/wp-cli folder first.
+mv ~/wp-cli/wp-cli-1.2.0.phar ~/wp-cli/wp
+chmod +x ~/wp-cli/wp
+
+WP_CLI_BIN_DIR=~/wp-cli composer behat
+```
+
+### Setting up the tests in Travis CI
+
+Basic rules for setting up the test framework with Travis CI:
+
+* `composer prepare-tests` needs to be called once per environment.
+* `linting and sniffing` is a static analysis, so it shouldn't depend on any specific environment. You should do this only once, as a separate stage, instead of per environment.
+* `composer behat || composer behat-rerun` causes the Behat tests to run in their entirety first, and in case their were failed scenarios, a second run is done with only the failed scenarios. This usually gets around intermittent issues like timeouts or similar.
+
+Here's a basic setup of how you can configure Travis CI to work with the test framework (extract):
+```yml
+install:
+  - composer install
+  - composer prepare-tests
+
+script:
+  - composer phpunit
+  - composer behat || composer behat-rerun
+
+jobs:
+  include:
+    - stage: sniff
+      script:
+        - composer lint
+        - composer phpcs
+      env: BUILD=sniff
+    - stage: test
+      php: 7.2
+      env: WP_VERSION=latest
+    - stage: test
+      php: 7.2
+      env: WP_VERSION=3.7.11
+    - stage: test
+      php: 7.2
+      env: WP_VERSION=trunk
+```
+
 #### WP-CLI version
 
 You can point the tests to a specific version ow WP-CLI through the `WP_CLI_BIN_DIR` constant: