From 9b433385bc54fd3ab4deba45cad367b90e9117bf Mon Sep 17 00:00:00 2001
From: Euphorbium <compositum@gmail.com>
Date: Sun, 25 Jan 2015 11:42:24 +0200
Subject: [PATCH] add random delay function between requests

---
 extension/devtools/views/SitemapScrapeConfig.html |  8 +++++++-
 extension/scripts/Controller.js                   | 14 ++++++++++++--
 extension/scripts/Scraper.js                      |  5 +++--
 3 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/extension/devtools/views/SitemapScrapeConfig.html b/extension/devtools/views/SitemapScrapeConfig.html
index 3ea00f90..f04bd4b0 100644
--- a/extension/devtools/views/SitemapScrapeConfig.html
+++ b/extension/devtools/views/SitemapScrapeConfig.html
@@ -6,6 +6,12 @@
 				<input type="text" class="form-control" name="requestInterval" id="requestInterval" placeholder="Request interval" value="2000">
 			</div>
 		</div>
+		<div class="form-group">
+			<label for="requestIntervalRandomness" class="col-lg-1 control-label">Request interval randomness (ms)</label>
+			<div class="col-lg-10">
+				<input type="text" class="form-control" name="requestIntervalRandomness" id="requestIntervalRandomness" placeholder="Request interval randomness" value="0">
+			</div>
+		</div>
 		<div class="form-group">
 			<label for="pageLoadDelay" class="col-lg-1 control-label">Page load delay (ms)</label>
 			<div class="col-lg-10">
@@ -23,4 +29,4 @@
 			</div>
 		</div>
 	</form>
-</div>
\ No newline at end of file
+</div>
diff --git a/extension/scripts/Controller.js b/extension/scripts/Controller.js
index 3caa4cab..0a05cb64 100644
--- a/extension/scripts/Controller.js
+++ b/extension/scripts/Controller.js
@@ -969,6 +969,16 @@ SitemapController.prototype = {
 						}
 					}
 				},
+				"requestIntervalRandomness": {
+					validators: {
+						notEmpty: {
+							message: 'The request interval randomness is required and cannot be empty'
+						},
+						numeric: {
+							message: 'The request interval randomness must be numeric'
+						},
+					}
+				},
 				"pageLoadDelay": {
 					validators: {
 						notEmpty: {
@@ -1042,7 +1052,7 @@ SitemapController.prototype = {
 			$("#viewport").html(dataPanel);
 
 			// display data
-			// Doing this the long way so there aren't xss vulnerubilites 
+			// Doing this the long way so there aren't xss vulnerubilites
 			// while working with data or with the selector titles
 			var $tbody = $("#sitemap-data tbody");
 			data.forEach(function (row) {
@@ -1407,7 +1417,7 @@ SitemapController.prototype = {
 			// remove from validator
 			var validator = this.getFormValidator();
 			validator.removeField($block.find("input"));
-			
+
 			$block.remove();
 		}
 	}
diff --git a/extension/scripts/Scraper.js b/extension/scripts/Scraper.js
index 2a5f25b9..71607880 100644
--- a/extension/scripts/Scraper.js
+++ b/extension/scripts/Scraper.js
@@ -5,6 +5,7 @@ Scraper = function (options) {
 	this.browser = options.browser;
 	this.resultWriter = null; // db instance for scraped data writing
 	this.requestInterval = parseInt(options.requestInterval);
+	this.requestIntervalRandomness = parseInt(options.requestIntervalRandomness);
 	this.pageLoadDelay = parseInt(options.pageLoadDelay);
 };
 
@@ -187,7 +188,7 @@ Scraper.prototype = {
 
 					var now = (new Date()).getTime();
 					// delay next job if needed
-					this._timeNextScrapeAvailable = now + this.requestInterval;
+					this._timeNextScrapeAvailable = now + this.requestInterval + Math.random()*this.requestIntervalRandomness;
 					if(now >= this._timeNextScrapeAvailable) {
 						this._run();
 					}
@@ -202,4 +203,4 @@ Scraper.prototype = {
 
 		}.bind(this));
 	}
-};
\ No newline at end of file
+};