-
\ No newline at end of file
+
diff --git a/extension/scripts/Controller.js b/extension/scripts/Controller.js
index 3caa4cab..0a05cb64 100644
--- a/extension/scripts/Controller.js
+++ b/extension/scripts/Controller.js
@@ -969,6 +969,16 @@ SitemapController.prototype = {
}
}
},
+ "requestIntervalRandomness": {
+ validators: {
+ notEmpty: {
+ message: 'The request interval randomness is required and cannot be empty'
+ },
+ numeric: {
+ message: 'The request interval randomness must be numeric'
+ },
+ }
+ },
"pageLoadDelay": {
validators: {
notEmpty: {
@@ -1042,7 +1052,7 @@ SitemapController.prototype = {
$("#viewport").html(dataPanel);
// display data
- // Doing this the long way so there aren't xss vulnerubilites
+ // Doing this the long way so there aren't xss vulnerubilites
// while working with data or with the selector titles
var $tbody = $("#sitemap-data tbody");
data.forEach(function (row) {
@@ -1407,7 +1417,7 @@ SitemapController.prototype = {
// remove from validator
var validator = this.getFormValidator();
validator.removeField($block.find("input"));
-
+
$block.remove();
}
}
diff --git a/extension/scripts/Scraper.js b/extension/scripts/Scraper.js
index 2a5f25b9..71607880 100644
--- a/extension/scripts/Scraper.js
+++ b/extension/scripts/Scraper.js
@@ -5,6 +5,7 @@ Scraper = function (options) {
this.browser = options.browser;
this.resultWriter = null; // db instance for scraped data writing
this.requestInterval = parseInt(options.requestInterval);
+ this.requestIntervalRandomness = parseInt(options.requestIntervalRandomness);
this.pageLoadDelay = parseInt(options.pageLoadDelay);
};
@@ -187,7 +188,7 @@ Scraper.prototype = {
var now = (new Date()).getTime();
// delay next job if needed
- this._timeNextScrapeAvailable = now + this.requestInterval;
+ this._timeNextScrapeAvailable = now + this.requestInterval + Math.random()*this.requestIntervalRandomness;
if(now >= this._timeNextScrapeAvailable) {
this._run();
}
@@ -202,4 +203,4 @@ Scraper.prototype = {
}.bind(this));
}
-};
\ No newline at end of file
+};