Add e2e tests (#119)

* Add e2e tests

* newline fixes

* fix typos

Author: s0ph1e

.gitignore

@@ -3,3 +3,4 @@
 node_modules
 npm-debug.log
 coverage
+test/e2e/results

package.json

@@ -5,7 +5,8 @@
   "readmeFilename": "README.md",
   "main": "index.js",
   "scripts": {
-    "test": "istanbul cover node_modules/mocha/bin/_mocha --dir ./coverage --report lcov -- -R spec --recursive --timeout 7000 ./test && npm run eslint",
+    "test": "istanbul cover node_modules/mocha/bin/_mocha --dir ./coverage --report lcov -- -R spec --recursive --timeout 7000 ./test/unit/ ./test/functional && npm run eslint",
+    "test-e2e": "node_modules/mocha/bin/_mocha --timeout 60000 ./test/e2e/*-test.js",
     "eslint": "eslint lib/** index.js"
   },
   "repository": {

test/e2e/e2e-test.js

@@ -0,0 +1,34 @@
+var should = require('should');
+var scraper = require('../../index');
+var URL = require('url');
+var fs = require('fs-extra');
+var _ = require('lodash');
+
+var urls = require('./urls.json');
+var options = require('./options.json');
+
+var resultDirname = __dirname + '/results';
+
+describe('E2E', function() {
+	before(function() {
+		fs.emptyDirSync(resultDirname);
+	});
+
+	after(function() {
+		console.log('Scraping completed. Go to ' + resultDirname + ' to check results');
+	});
+
+	urls.forEach(function(url) {
+		describe(url, function() {
+			it('should be successfully scraped', function() {
+				var scraperOptions = _.clone(options);
+				var hostname = URL.parse(url).hostname;
+				scraperOptions.directory = resultDirname + '/' + hostname;
+				scraperOptions.urls = [ { url: url, filename: 'index.html' } ];
+				return scraper.scrape(scraperOptions).then(function(result) {
+					result.should.be.ok();
+				});
+			});
+		});
+	});
+});

test/e2e/options.json

@@ -0,0 +1,7 @@
+{
+  "request": {
+    "headers": {
+      "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.115 Safari/537.36"
+    }
+  }
+}

test/e2e/urls.json

@@ -0,0 +1,15 @@
+[
+  "https://nodejs.org/docs/latest/api/documentation.html",
+  "https://www.npmjs.com/",
+  "https://www.airbnb.com/",
+  "http://www.ebay.com/",
+  "http://www.booking.com/",
+  "https://www.kinopoisk.ru/",
+  "https://www.linkedin.com/",
+  "http://www.apple.com/",
+  "https://habrahabr.ru/post/310794/",
+  "http://www.caminitodelrey.info/",
+  "http://eslint.org/docs/rules/",
+  "http://www.lostfilm.tv/",
+  "https://www.antipenko.pp.ua/"
+]