New fetures

README.md

@@ -1,13 +1,27 @@
-# Web Scraper
-Web Scraper is a chrome browser extension built for data extraction from web 
+# Web Scraper Plus
+Web Scraper Plus is a chrome browser extension built for data extraction from web 
 pages. Using this extension you can create a plan (sitemap) how a web site 
 should be traversed and what should be extracted. Using these sitemaps the 
 Web Scraper will navigate the site accordingly and extract all data. Scraped 
 data later can be exported as CSV.
 
-Install the extension from [Chrome store] [chrome-store]
+Install the extension from [chrome-store]
 
-### Features
+Document for new features: [wiki]
+
+#### This tool is forked form [Web-Scraper] with many more features
+
+### New Features
+ 1. [CLI Support]: Start scraping from CMD/Terminal
+ 2. [MySQL Support]: Support MySQL database (v5.7+)
+ 3. [Anti Lazy-Loading]: Anti Lazy-Loading feature on pages
+ 4. [Data Filter]: Support user defined JS code for data preprocess and much more
+ 5. [Distinct]: Remove dulplicate data before the end of every task.
+ 6. [Custom Columns]: Define the columns you want to display, please use this feature together with [Data Filter]
+ 7. [Easy Scrape]: Create & scrape sitemap in a more easily way. (Based on https://github.com/aagiss)
+ 8. Random Interval: Add a random delay between requests. (Provided by https://github.com/Euphorbium)
+
+### Features(Forked from original work)
 
  1. Scrape multiple pages
  2. Sitemaps and scraped data are stored in browsers local storage or in CouchDB
@@ -20,42 +34,31 @@ Install the extension from [Chrome store] [chrome-store]
 
 ### Help
 
- Documentation and tutorials are available on [webscraper.io] [webscraper.io]
+ Basic documentation and tutorials are available on [webscraper.io]
 
- Ask for help, submit bugs, suggest features on [google groups] [google-groups]
-
- Submit bugs and suggest features on [bug tracker] [github-issues]
+ Submit bugs and suggest features on [github-issues]
 
 #### Bugs
 When submitting a bug please attach an exported sitemap if possible.
 
 ## License
 LGPLv3
 
-## Changelog
-
-### v0.2
- * Added Element click selector
- * Added Element scroll down selector
- * Added Link popup selector
- * Improved table selector to work with any html markup
- * Added Image download
- * Added keyboard shortcuts when selecting elements
- * Added configurable delay before using selector
- * Added configurable delay between page visiting
- * Added multiple start url configuration
- * Added form field validation
- * Fixed a lot of bugs
-
-### v0.1.3
- * Added Table selector
- * Added HTML selector
- * Added HTML attribute selector
- * Added data preview
- * Added ranged start urls
- * Fixed bug which made selector tree not to show on some operating systems
-
- [chrome-store]: https://chrome.google.com/webstore/detail/web-scraper/jnhgnonknehpejjnehehllkliplmbmhn
+ [Web-Scraper]: https://github.com/martinsbalodis/web-scraper-chrome-extension
+ [chrome-store]: https://chrome.google.com/webstore/detail/pbbfbmlnpackgeofecdfncmmdbodkhma
  [webscraper.io]: http://webscraper.io/
- [google-groups]: https://groups.google.com/forum/#!forum/web-scraper
- [github-issues]: https://github.com/martinsbalodis/web-scraper-chrome-extension/issues
+ [github-issues]: https://github.com/hejiheji001/web-scraper-chrome-extension/issues
+ [wiki]: https://github.com/hejiheji001/web-scraper-chrome-extension/wiki
+[MySQL Support]: https://github.com/hejiheji001/web-scraper-chrome-extension/wiki/MySQL-Support
+
+[CLI Support]: https://github.com/hejiheji001/web-scraper-chrome-extension/wiki/CLI-Support
+
+[Anti Lazy-Loading]: https://github.com/hejiheji001/web-scraper-chrome-extension/wiki/Anti-Lazy-Loading
+
+[Data Filter]: https://github.com/hejiheji001/web-scraper-chrome-extension/wiki/Data-Filter
+
+[Distinct]: https://github.com/hejiheji001/web-scraper-chrome-extension/wiki/Distinct
+
+[Custom Columns]: https://github.com/hejiheji001/web-scraper-chrome-extension/wiki/Custom-Columns
+
+[Easy Scrape]: https://github.com/hejiheji001/web-scraper-chrome-extension/wiki/Easy-Scrape

extension/background_page/background_script.js

@@ -27,12 +27,22 @@ var sendToActiveTab = function(request, callback) {
 	});
 };
 
+var currentChildURLs=null;
+
 chrome.runtime.onMessage.addListener(
 	function (request, sender, sendResponse) {
 
 		console.log("chrome.runtime.onMessage", request);
 
-		if (request.createSitemap) {
+		if (request.setCurrentChildURLs) {
+			currentChildURLs=request.urls;
+			sendResponse(request);
+			return true;
+		}else if (request.getCurrentChildURLs) {
+			request.urls=currentChildURLs;
+			sendResponse(request);
+			return true;
+		}else if (request.createSitemap) {
 			store.createSitemap(request.sitemap, sendResponse);
 			return true;
 		}
@@ -51,16 +61,22 @@ chrome.runtime.onMessage.addListener(
 		else if (request.sitemapExists) {
 			store.sitemapExists(request.sitemapId, sendResponse);
 			return true;
+		}else if (request.findSitemap) {
+			store.findSitemap(request.sitemapId, sendResponse);
+			return true;
 		}
 		else if (request.getSitemapData) {
+			store.getSitemapData(new Sitemap(request.sitemap), sendResponse);
 			store.getSitemapData(new Sitemap(request.sitemap), sendResponse);
 			return true;
 		}
-		else if (request.scrapeSitemap) {
+		else if (request.scrapeSitemap) {//TODO
 			var sitemap = new Sitemap(request.sitemap);
 			var queue = new Queue();
 			var browser = new ChromePopupBrowser({
-				pageLoadDelay: request.pageLoadDelay
+				pageLoadDelay: request.pageLoadDelay,
+				scrollToBottom: request.scrollToBottom,
+				urls: sitemap.getStartUrls()
 			});
 
 			var scraper = new Scraper({
@@ -72,8 +88,8 @@ chrome.runtime.onMessage.addListener(
 			});
 
 			try {
-				scraper.run(function () {
-					browser.close();
+				const callback = function(){
+					//browser.close();
 					var notification = chrome.notifications.create("scraping-finished", {
 						type: 'basic',
 						iconUrl: 'assets/images/icon128.png',
@@ -83,7 +99,14 @@ chrome.runtime.onMessage.addListener(
 						// notification showed
 					});
 					sendResponse();
-				});
+				}
+
+				const finalize = function(finalResult){
+					store.removeDuplicate(sitemap._id, request.distinct, finalResult);
+					callback();
+				}
+
+				scraper.run(finalize);
 			}
 			catch (e) {
 				console.log("Scraper execution cancelled".e);

extension/content_script/content_script.js

@@ -17,6 +17,7 @@ chrome.runtime.onMessage.addListener(
 			console.log("received data-preview extraction request", request);
 			var extractor = new DataExtractor(request);
 			var deferredData = extractor.getSingleSelectorData(request.parentSelectorIds, request.selectorId);
+
 			deferredData.done(function(data){
 				console.log("dataextractor data", data);
 				sendResponse(data);
@@ -36,6 +37,9 @@ chrome.runtime.onMessage.addListener(
 			});
 
 			return true;
+		}else if(request.notice){
+			console.log("Notification: ", request.message);
+
 		}
 	}
 );

extension/content_script/scrollToBottom.js

@@ -0,0 +1,13 @@
+chrome.runtime.onMessage.addListener(function(message, sender, sendResponse) {
+	if(message.run){
+		var i = 0;
+		var x = setInterval(function(){
+			window.scrollTo(0, i * 100);
+			i++;
+			if(i * 100 >= document.body.scrollHeight){
+				clearInterval(x);
+				chrome.runtime.sendMessage({antilazyloading: true});
+			}
+		}, 100);
+	}
+});

extension/devtools/devtools_scraper_panel.html

@@ -34,6 +34,8 @@
 	<script src="../scripts/Controller.js"></script>
 
 	<script src="../scripts/App.js"></script>
+
+	<script src="../scripts/RunTask.js"></script>
 </head>
 <body></body>
 </html>