test: visual regression testing (#885)

* first pass at visual regression testing

* updating package-lock.json

* updating the baseline

* adding http-server for local testing

I couldn't figure out a way to kill the npm start process in the onComplete callback for web driver. Adding http-server, starting it in onPrepare, storing the process as a variable, and then killing it in the onComplete callback works well.

* working on creating dynamic tests

* adding a test for pfe-accordion

* updating travis file to run just e2e second

* updating pfe-accordion baseline images

* new baseline images, more tests, and updated configuration

* pfe-collapse, pfe-content-set, pfe-cta, pfe-datetime tests

* more tests and baseline images

* using local versions of webcomponentsjs and require

Trying to cut down on external dependencies for testing

* moving webcomponentsjs and requirejs to dependencies

* making webcomponentsjs and requirejs devDependencies again

* copying webcomponentsjs and requirejs in the build step

* using webcomponentsjs and requirejs from the examples directory

* using local images instead of images from the internet

* updating spandx config to the root doesn't point to just node_modules

* increasing e2e test timeout to 30 seconds and increasing the maxInstances to 7

* updating the location of the webcomponent polyfills and require

* creating a simplified e2e demo page for pfe-icon

* updating and creating e2e tests

* updated and new baseline images

* updating the location of the web component polyfills and require.js in the generator

* adding more wait time in the autocomplete e2e test

* updating demo page polyfills and require and updating baseline images

* trying a lower number of max instances

* more baseline updates

* updating pfe-health-index IE11 baseline

* updating pfe-cta IE11 baseline

* adding 500x500 kitten

* baseline updates and test adjustments

* package-lock updates

* increase the tolerance in the pfe-toast test

Author: kylebuch8

.gitignore

@@ -1,6 +1,7 @@
 # Development files
 node_modules
 tmp
+.tmp
 
 # Generator config file
 project.conf.json
@@ -16,6 +17,7 @@ doc/index.html
 # Logs
 lerna-debug.log
 npm-debug.log*
+local.log
 
 # Local files
 .env

.travis.yml

@@ -7,8 +7,12 @@ os:
 addons:
   firefox: latest
   chrome: stable
+  browserstack:
+    username: BROWSERSTACK_USER
+    access_key: BROWSERSTACK_KEY
 script:
   - npm test -- --verbose
+  - npm run e2e -- --verbose
 cache:
   directories:
     - node_modules

elements/pfe-accordion/demo/index.html

@@ -21,9 +21,9 @@
     <link rel="stylesheet" href="../../pfe-styles/dist/pfe-layouts.css" />
 
     <!-- uncomment the es5-adapter if you're using the umd version -->
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/webcomponentsjs/2.3.0/custom-elements-es5-adapter.js"></script>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/webcomponentsjs/2.3.0/webcomponents-bundle.js"></script>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.5/require.min.js"></script>
+    <script src="/examples/node_modules/@webcomponents/webcomponentsjs/custom-elements-es5-adapter.js"></script>
+    <script src="/examples/node_modules/@webcomponents/webcomponentsjs/webcomponents-bundle.js"></script>
+    <script src="/examples/node_modules/requirejs/require.js"></script>
 
     <script>require([
       '../dist/pfe-accordion.umd.js',
@@ -105,7 +105,7 @@ <h2>Accordion component</h2>
       </header>
       <article>
         <pfe-accordion>
-          <pfe-accordion-header>
+          <pfe-accordion-header id="first-header">
             <h3>Nested panel example</h3>
           </pfe-accordion-header>
           <pfe-accordion-panel>

elements/pfe-accordion/test/pfe-accordion_e2e.js

@@ -0,0 +1,23 @@
+const element = require("../package.json").pfelement.elementName;
+
+describe(element, () => {
+  let accordion;
+
+  before(() => {
+    browser.url(`/elements/${element}/demo`);
+
+    accordion = $(element);
+    accordion.$(function() {
+      this.toggle(0);
+    });
+    browser.pause(1000);
+  });
+
+  it("should take a screenshot", () => {
+    browser.saveScreen(element);
+  });
+
+  it("should compare to the baseline", () => {
+    expect(browser.checkScreen(element)).toBeLessThan(1.25);
+  });
+});

elements/pfe-autocomplete/demo/index.html

@@ -21,9 +21,9 @@
     <link href="../../pfelement/dist/pfelement.min.css" rel="stylesheet">
 
     <!-- uncomment the es5-adapter if you're using the umd version -->
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/webcomponentsjs/2.3.0/custom-elements-es5-adapter.js"></script>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/webcomponentsjs/2.3.0/webcomponents-bundle.js"></script>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.5/require.min.js"></script>
+    <script src="/examples/node_modules/@webcomponents/webcomponentsjs/custom-elements-es5-adapter.js"></script>
+    <script src="/examples/node_modules/@webcomponents/webcomponentsjs/webcomponents-bundle.js"></script>
+    <script src="/examples/node_modules/requirejs/require.js"></script>
     <script>require([
       '../dist/pfe-autocomplete.umd.js'
     ])</script>

elements/pfe-autocomplete/test/pfe-autocomplete_e2e.js

@@ -0,0 +1,29 @@
+const element = require("../package.json").pfelement.elementName;
+
+describe(element, () => {
+  let autocomplete;
+  let input
+
+  before(() => {
+    browser.url(`/elements/${element}/demo`);
+    autocomplete = $(element);
+    input = $(element + " input");
+    
+    browser.execute(function (element) {
+      element.focus();
+    }, input);
+    
+    browser.keys(['i']);
+    browser.pause(1000);
+    browser.keys(['Down arrow']);
+    browser.pause(1000);
+  });
+
+  it("should take a screenshot", () => {
+    browser.saveScreen(element);
+  });
+
+  it("should compare to the baseline", () => {
+    expect(browser.checkScreen(element)).toBeLessThan(1.5);
+  });
+});

elements/pfe-avatar/demo/index.html

@@ -22,9 +22,9 @@
     <script src="https://cdnjs.cloudflare.com/ajax/libs/vue/2.5.2/vue.min.js"></script>
 
     <!-- uncomment the es5-adapter if you're using the umd version -->
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/webcomponentsjs/2.3.0/custom-elements-es5-adapter.js"></script>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/webcomponentsjs/2.3.0/webcomponents-bundle.js"></script>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.5/require.min.js"></script>
+    <script src="/examples/node_modules/@webcomponents/webcomponentsjs/custom-elements-es5-adapter.js"></script>
+    <script src="/examples/node_modules/@webcomponents/webcomponentsjs/webcomponents-bundle.js"></script>
+    <script src="/examples/node_modules/requirejs/require.js"></script>
 
     <script>require([
       '../dist/pfe-avatar.umd.js'

elements/pfe-avatar/test/pfe-avatar_e2e.js

@@ -0,0 +1,15 @@
+const element = require("../package.json").pfelement.elementName;
+
+describe(element, () => {
+  before(() => {
+    browser.url(`/elements/${element}/demo`);
+  });
+
+  it("should take a screenshot", () => {
+    browser.saveScreen(element);
+  });
+
+  it("should compare to the baseline", () => {
+    expect(browser.checkScreen(element)).toBeLessThan(1.25);
+  });
+});

elements/pfe-badge/demo/index.html

@@ -8,9 +8,9 @@
   <title>PatternFly Element | pfe-badge Demo</title>
 
   <!-- uncomment the es5-adapter if you're using the umd version -->
-  <script src="https://cdnjs.cloudflare.com/ajax/libs/webcomponentsjs/2.2.10/custom-elements-es5-adapter.js"></script>
-  <script src="https://cdnjs.cloudflare.com/ajax/libs/webcomponentsjs/2.2.10/webcomponents-bundle.js"></script>
-  <script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.5/require.min.js"></script>
+  <script src="/examples/node_modules/@webcomponents/webcomponentsjs/custom-elements-es5-adapter.js"></script>
+  <script src="/examples/node_modules/@webcomponents/webcomponentsjs/webcomponents-bundle.js"></script>
+  <script src="/examples/node_modules/requirejs/require.js"></script>
   <script>require(['../dist/pfe-badge.umd.js'])</script>
 
   <style>

elements/pfe-badge/test/pfe-badge_e2e.js

@@ -0,0 +1,15 @@
+const element = require("../package.json").pfelement.elementName;
+
+describe(element, () => {
+  before(() => {
+    browser.url(`/elements/${element}/demo`);
+  });
+
+  it("should take a screenshot", () => {
+    browser.saveScreen(element);
+  });
+
+  it("should compare to the baseline", () => {
+    expect(browser.checkScreen(element)).toBeLessThan(1.25);
+  });
+});