Merge remote-tracking branch 'upstream/dev'

Author: aivus

.travis.yml

@@ -0,0 +1,4 @@
+language: node_js
+node_js:
+  - "0.11"
+  - "0.10"

README.md

@@ -1,67 +1,93 @@
 ##Introduction
-Node.js module for website's scraping with images, css, js, etc. Uses cheerio, request, bluebird, fs-extra, underscore.
+Node.js module for website's scraping with images, css, js, etc. 
+
+[![Build Status](https://img.shields.io/travis/s0ph1e/node-website-scraper/master.svg?style=flat)](https://travis-ci.org/s0ph1e/node-website-scraper)
+[![Code Climate](https://img.shields.io/codeclimate/github/s0ph1e/node-website-scraper.svg?style=flat)](https://codeclimate.com/github/s0ph1e/node-website-scraper)
+[![Version](https://img.shields.io/npm/v/website-scraper.svg?style=flat)](https://www.npmjs.org/package/website-scraper)
+[![Downloads](https://img.shields.io/npm/dm/website-scraper.svg?style=flat)](https://www.npmjs.org/package/website-scraper)
+[![Dependency Status](https://david-dm.org/s0ph1e/node-website-scraper.svg?style=flat)](https://david-dm.org/s0ph1e/node-website-scraper)
+
+[![NPM Stats](https://nodei.co/npm/website-scraper.png?downloadRank=true&stars=true)](https://www.npmjs.org/package/website-scraper)
 
 ##Installation
 `npm install website-scraper`
 
 ##Usage
 ```javascript
 var scraper = require('website-scraper'); 
-scraper.scrape({
+var options = {
   url: 'http://nodejs.org/',
-  path: '/path/to/save/',
-}, function (error, result){
+  directory: '/path/to/save/',
+};
+
+// with callback
+scraper.scrape(options, function (error, result) {
+	/* some code here */
+});
+
+// or with promise
+scraper.scrape(options).then(function (result) {
 	/* some code here */
 });
 ```
 
 ##API
 ### scrape(options, callback)
-Makes request to `url` and saves all files found with `srcToLoad` to `path`. 
+Makes request to `url` and saves all files found with `srcToLoad` to `directory`. 
 
 **options** - object containing next options:
 
  - `url:` url to load *(required)*
- - `path:` path to save loaded files *(required)*
+ - `directory:` path to save loaded files *(required)*
+ - `paths:` array of objects, contains urls or relative paths to load and filenames for them (if is not set only `url` will be loaded) *(optional, see example below)*
  - `log:` boolean indicates whether to write the log to console *(optional, default: false)*
  - `indexFile:` filename for index page *(optional, default: 'index.html')*
  - `srcToLoad:` array of objects to load, specifies selectors and attribute values to select files for loading *(optional, see default value in `lib/defaults.js`)*
- - `directories:` array of objects, specifies relative directories for extensions. If `null` all files will be saved to `path` *(optional, see example below)*
+ - `subdirectories:` array of objects, specifies subdirectories for extensions. If `null` all files will be saved to `directory` *(optional, see example below)*
 
 
 **callback** - callback function *(optional)*, includes following parameters:
 
   - `error:` if error - `Error object`, if success - `null`
-  - `result:` if error - `null`, if success - object containing:
-    - `html:` html code of index page
+  - `result:` if error - `null`, if success - array if objects containing:
+    - `url:` url of loaded page
+    - `filename:` absolute filename where page was saved
 
 
 ##Examples
-Let's scrape [http://nodejs.org/](http://nodejs.org/) with images, css, js files and save them to `/path/to/save/`. Index page will be named 'myIndex.html', files will be separated into directories:
+Let's scrape some pages from [http://nodejs.org/](http://nodejs.org/) with images, css, js files and save them to `/path/to/save/`. 
+Imagine we want to load:
+  - [Home page](http://nodejs.org/) to `index.html`
+  - [About page](http://nodejs.org/about/) to `about.html`
+  - [Blog](http://blog.nodejs.org/) to `blog.html`
+  
+and separate files into directories:
 
-  - `img` for .jpg, .png (full path `/path/to/save/img`) 
+  - `img` for .jpg, .png, .svg (full path `/path/to/save/img`) 
   - `js` for .js (full path `/path/to/save/js`)
   - `css` for .css (full path `/path/to/save/css`)
-  - `font` for .ttf, .woff, .eot, .svg (full path `/path/to/save/font`)
 
 ```javascript
 scraper.scrape({
   url: 'http://nodejs.org/',
-  path: '/path/to/save',
-  indexFile: 'myIndex.html',
+  directory: '/path/to/save',
+  paths: [
+    {path: '/', filename: 'index.html'},
+    {path: '/about', filename: 'about.html'},
+    {url: 'http://blog.nodejs.org/', filename: 'blog.html'}
+  ],
+  subdirectories: [
+    {directory: 'img', extensions: ['.jpg', '.png', '.svg']},
+    {directory: 'js', extensions: ['.js']},
+    {directory: 'css', extensions: ['.css']}
+  ],
   srcToLoad: [
     {selector: 'img', attr: 'src'},
     {selector: 'link[rel="stylesheet"]', attr: 'href'},
     {selector: 'script', attr: 'src'}
-  ],
-  directories: [
-    {directory: 'img', extensions: ['.jpg', '.png']},
-    {directory: 'js', extensions: ['.js']},
-    {directory: 'css', extensions: ['.css']},
-    {directory: 'fonts', extensions: ['.ttf', '.woff', '.eot', '.svg']}
   ]
-}, function (error, result){
-	console.log(result);
+}).then(function (result) {
+  console.log(result);
 });
 ```
 

lib/defaults.js renamed to lib/config/defaults.js

@@ -35,7 +35,7 @@ var config = {
       attr: 'href'
     },
   ],
-  directories: [
+  subdirectories: [
     {
       directory: 'images',
       extensions: ['.png', '.jpg', '.jpeg', '.gif']
@@ -55,4 +55,4 @@ var config = {
   ]
 };
 
-module.exports = config;
+module.exports = config;