Skip to content

Commit 1e85c12

Browse files
jwillmerjwillmer
committed
Refactored sitemap start url GUI
Fixed tests close #4
1 parent cb3d802 commit 1e85c12

File tree

12 files changed

+56
-149
lines changed

12 files changed

+56
-149
lines changed

‎extension/devtools/views/SitemapCreate.html‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,10 @@
77
</div>
88
</div>
99
<div class="form-group start-url-block">
10-
<label class="col-lg-1 control-label">Start URL</label>
10+
<label class="col-lg-1 control-label">Start URLs</label>
1111
<div class="col-lg-10">
1212
<div class="input-group">
13-
<input type="text" class="form-control input-start-url" name="startUrl[]" placeholder="URL">
14-
<span class="input-group-btn">
15-
<button class="btn btn-default remove-start-url" type="button">-</button>
16-
<button class="btn btn-default add-extra-start-url" type="button">+</button>
17-
</span>
13+
<textarea rows="5" cols="100" class="form-control input-start-url" name="startUrls" placeholder="Comma seperated URL list"></textarea>
1814
</div>
1915
</div>
2016
</div>

‎extension/devtools/views/SitemapEditMetadata.html‎

Lines changed: 8 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -7,36 +7,14 @@
77
<input type="text" class="form-control" name="_id" id="edit_sitemap_id" placeholder="Sitemap name" value="{{_id}}">
88
</div>
99
</div>
10-
{{#startUrl.push}}
11-
{{#startUrl}}
12-
<div class="form-group start-url-block">
13-
<label class="col-lg-1 control-label">Start URL</label>
14-
<div class="col-lg-10">
15-
<div class="input-group">
16-
<input type="text" class="form-control input-start-url" name="startUrl[]" placeholder="URL" value="{{.}}">
17-
<span class="input-group-btn">
18-
<button class="btn btn-default remove-start-url" type="button">-</button>
19-
<button class="btn btn-default add-extra-start-url" type="button">+</button>
20-
</span>
21-
</div>
22-
</div>
23-
</div>
24-
{{/startUrl}}
25-
{{/startUrl.push}}
26-
{{^startUrl.push}}
27-
<div class="form-group start-url-block">
28-
<label class="col-lg-1 control-label">Start URL</label>
29-
<div class="col-lg-10">
30-
<div class="input-group">
31-
<input type="text" class="form-control input-start-url" name="startUrl[]" placeholder="URL" value="{{startUrl}}">
32-
<span class="input-group-btn">
33-
<button class="btn btn-default remove-start-url" type="button">-</button>
34-
<button class="btn btn-default add-extra-start-url" type="button">+</button>
35-
</span>
36-
</div>
37-
</div>
38-
</div>
39-
{{/startUrl.push}}
10+
<div class="form-group start-url-block">
11+
<label class="col-lg-1 control-label">Start URL</label>
12+
<div class="col-lg-10">
13+
<div class="input-group">
14+
<textarea rows="5" cols="100" class="form-control input-start-url" name="startUrls" placeholder="URL List"></textarea>
15+
</div>
16+
</div>
17+
</div>
4018

4119
<div class="col-lg-10 col-lg-offset-1">
4220
Supported URL patterns:<br>

‎extension/devtools/views/SitemapListItem.html‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,9 @@
11
<tr>
22
<td class="id">{{_id}}</td>
33
<td>
4-
{{#startUrl.push}}
5-
{{#startUrl}}
4+
{{#startUrls}}
65
{{.}},
7-
{{/startUrl}}
8-
{{/startUrl.push}}
9-
{{^startUrl.push}}
10-
{{startUrl}}
11-
{{/startUrl.push}}
6+
{{/startUrls}}
127
</td>
138
<td>
149
<button action="browse-sitemap-data" type="button" class="btn btn-primary btn-xs">Browse</button>

‎extension/devtools/views/SitemapStartUrlField.html‎

Lines changed: 0 additions & 13 deletions
This file was deleted.

‎extension/scripts/ChromePopupBrowser.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ ChromePopupBrowser.prototype = {
7878
chrome.tabs.sendMessage(tab.id, message, function (data, selectors) {
7979
console.log("extracted data from web page", data);
8080

81-
if (selectors) {
81+
if (selectors && scope) {
8282
// table selector can dynamically add columns (addMissingColumns Feature)
8383
scope.scraper.sitemap.selectors = selectors;
8484
}

‎extension/scripts/Controller.js‎

Lines changed: 18 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ SitemapController.prototype = {
3737
'SitemapList',
3838
'SitemapListItem',
3939
'SitemapCreate',
40-
'SitemapStartUrlField',
4140
'SitemapImport',
4241
'SitemapExport',
4342
'SitemapBrowseData',
@@ -197,13 +196,7 @@ SitemapController.prototype = {
197196
},
198197
"#edit-selector button[action=preview-selector-data]": {
199198
click: this.previewSelectorDataFromSelectorEditing
200-
},
201-
"button.add-extra-start-url": {
202-
click: this.addStartUrl
203-
},
204-
"button.remove-start-url": {
205-
click: this.removeStartUrl
206-
}
199+
}
207200
});
208201
this.showSitemaps();
209202
}.bind(this));
@@ -247,22 +240,6 @@ SitemapController.prototype = {
247240
}
248241
},
249242

250-
/**
251-
* Simple info popup for sitemap start url input field
252-
*/
253-
initMultipleStartUrlHelper: function () {
254-
$("#startUrl")
255-
.popover({
256-
title: 'Multiple start urls',
257-
html: true,
258-
content: "You can create ranged start urls like this:<br />http://example.com/[1-100].html",
259-
placement: 'bottom'
260-
})
261-
.blur(function () {
262-
$(this).popover('hide');
263-
});
264-
},
265-
266243
/**
267244
* Returns bootstrapValidator object for current form in viewport
268245
*/
@@ -318,14 +295,23 @@ SitemapController.prototype = {
318295
}
319296
}
320297
},
321-
"startUrl[]": {
298+
"startUrls": {
322299
validators: {
323300
notEmpty: {
324301
message: 'The start URL is required and cannot be empty'
325302
},
326-
uri: {
327-
allowLocal: true,
328-
message: 'The start URL is not a valid URL'
303+
callback: {
304+
message: 'The start URLs are not valid. Please use "," as a seperator.',
305+
callback: function (value, validator) {
306+
function isUrlValid(url) {
307+
return /^(https?|s?ftp):\/\/(((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:)*@)?(((\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5]))|((([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))\.)+(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))\.?)(:\d*)?)(\/((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)+(\/(([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)*)*)?)?(\?((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)|[\uE000-\uF8FF]|\/|\?)*)?(#((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)|\/|\?)*)?$/i.test(url);
308+
}
309+
var isValid = true;
310+
value.split(',').map((item) => item.trim()).each(function (url) {
311+
isValid = isUrlValid(url) ? isValid : false;
312+
});
313+
return isValid;
314+
}
329315
}
330316
}
331317
}
@@ -337,7 +323,6 @@ SitemapController.prototype = {
337323
this.setActiveNavigationButton('create-sitemap-create');
338324
var sitemapForm = ich.SitemapCreate();
339325
$("#viewport").html(sitemapForm);
340-
this.initMultipleStartUrlHelper();
341326
this.initSitemapValidation();
342327

343328
return true;
@@ -426,20 +411,11 @@ SitemapController.prototype = {
426411

427412
var id = $("#viewport form input[name=_id]").val();
428413
var $startUrlInputs = $("#viewport form .input-start-url");
429-
var startUrl;
430-
if($startUrlInputs.length === 1) {
431-
startUrl = $startUrlInputs.val();
432-
}
433-
else {
434-
startUrl = [];
435-
$startUrlInputs.each(function(i, element) {
436-
startUrl.push($(element).val());
437-
});
438-
}
414+
var startUrls = $startUrlInputs.val().split(',').map((item) => item.trim());
439415

440416
return {
441417
id:id,
442-
startUrl:startUrl
418+
startUrls:startUrls
443419
};
444420
},
445421

@@ -461,7 +437,7 @@ SitemapController.prototype = {
461437
else {
462438
var sitemap = new Sitemap({
463439
_id: sitemapData.id,
464-
startUrl: sitemapData.startUrl,
440+
startUrls: sitemapData.startUrls,
465441
selectors: []
466442
});
467443
this.store.createSitemap(sitemap, function (sitemap) {
@@ -508,7 +484,6 @@ SitemapController.prototype = {
508484
var sitemap = this.state.currentSitemap;
509485
var $sitemapMetadataForm = ich.SitemapEditMetadata(sitemap);
510486
$("#viewport").html($sitemapMetadataForm);
511-
this.initMultipleStartUrlHelper();
512487
this.initSitemapValidation();
513488

514489
return true;
@@ -532,7 +507,7 @@ SitemapController.prototype = {
532507
}
533508

534509
// change data
535-
sitemap.startUrl = sitemapData.startUrl;
510+
sitemap.startUrls = sitemapData.startUrls;
536511

537512
// just change sitemaps url
538513
if (sitemapData.id === sitemap._id) {
@@ -1487,32 +1462,5 @@ SitemapController.prototype = {
14871462
$(this).remove();
14881463
});
14891464
});
1490-
},
1491-
/**
1492-
* Add start url to sitemap creation or editing form
1493-
* @param button
1494-
*/
1495-
addStartUrl: function(button) {
1496-
1497-
var $startUrlInputField = ich.SitemapStartUrlField();
1498-
$("#viewport .start-url-block:last").after($startUrlInputField);
1499-
var validator = this.getFormValidator();
1500-
validator.addField($startUrlInputField.find("input"));
1501-
},
1502-
/**
1503-
* Remove start url from sitemap creation or editing form.
1504-
* @param button
1505-
*/
1506-
removeStartUrl: function(button) {
1507-
1508-
var $block = $(button).closest(".start-url-block");
1509-
if($("#viewport .start-url-block").length > 1) {
1510-
1511-
// remove from validator
1512-
var validator = this.getFormValidator();
1513-
validator.removeField($block.find("input"));
1514-
1515-
$block.remove();
1516-
}
15171465
}
15181466
};

‎extension/scripts/Job.js‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,12 @@ Job.prototype = {
6868
}
6969
this.dataItems.push(result);
7070
}
71-
// table selector can dynamically add columns (addMissingColumns Feature)
72-
sitemap.selectors = this.scraper.sitemap.selectors;
71+
72+
if (sitemap) {
73+
// table selector can dynamically add columns (addMissingColumns Feature)
74+
sitemap.selectors = this.scraper.sitemap.selectors;
75+
}
76+
7377
console.log(job);
7478
callback(job);
7579
}.bind(this), this);

‎extension/scripts/Sitemap.js‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -62,12 +62,7 @@ Sitemap.prototype = {
6262

6363
getStartUrls: function() {
6464

65-
var startUrls = this.startUrl;
66-
// single start url
67-
if(this.startUrl.push === undefined) {
68-
startUrls = [startUrls];
69-
}
70-
65+
var startUrls = this.startUrls;
7166
startUrls = DatePatternSupport.expandUrl(startUrls);
7267

7368
var nextUrls = function (url) {

‎tests/FakeStore.js‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,5 +13,9 @@ FakeStore.prototype = {
1313

1414
initSitemapDataDb: function (sitemapId, callback) {
1515
callback(this);
16-
}
16+
},
17+
18+
saveSitemap: function (sitemap, callback) {
19+
callback(this);
20+
}
1721
};

‎tests/spec/DataExtractSpec.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1076,7 +1076,7 @@ describe("DataExtractor", function () {
10761076
"selector": "td:nth-of-type(9) a"
10771077
}
10781078
],
1079-
"startUrl": "http://www.centos.org/modules/tinycontent/index.php?id=30",
1079+
"startUrls": "http://www.centos.org/modules/tinycontent/index.php?id=30",
10801080
"_id": "centos-mirrors2"
10811081
});
10821082

0 commit comments

Comments
 (0)