Skip to content

Commit 50f451e

Browse files
gorhillgorhill
committed
new set of URI tools beter suited for HTTPSB; Mozilla's PSL; punycode
1 parent a52056e commit 50f451e

File tree

16 files changed

+8117
-160
lines changed

16 files changed

+8117
-160
lines changed

‎assets/thirdparties/mxr.mozilla.org/effective_tld_names.dat‎

Lines changed: 7447 additions & 0 deletions
Large diffs are not rendered by default.

‎background.html‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,16 @@
44
<title>HTTP Switchboard</title>
55
</head>
66
<body>
7-
<script src="lib/uri.min.js"></script>
7+
<script src="lib/punycode.min.js"></script>
8+
<script src="lib/publicsuffixlist.min.js"></script>
89

910
<script src="js/types.js"></script>
11+
<script src="js/uritools.js"></script>
1012
<script src="js/async.js"></script>
1113
<script src="js/lists.js"></script>
1214
<script src="js/background.js"></script>
1315
<script src="js/httpsb.js"></script>
1416
<script src="js/reqstats.js"></script>
15-
<script src="js/cacher.js"></script>
1617
<script src="js/cookies.js"></script>
1718
<script src="js/inject.js"></script>
1819
<script src="js/profiler.js"></script>

‎js/async.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -198,8 +198,8 @@ function onMessageHandler(request, sender, callback) {
198198

199199
case 'contentHasLocalStorage':
200200
// `blocked` aka `response`
201-
response = HTTPSB.blacklisted(request.url, 'cookie', getHostnameFromURL(request.url));
202-
recordFromPageUrl(request.url, 'cookie', getRootURLFromURL(request.url) + '/{localStorage}', response);
201+
response = HTTPSB.blacklisted(request.url, 'cookie', uriTools.hostnameFromURI(request.url));
202+
recordFromPageUrl(request.url, 'cookie', uriTools.rootURLFromURI(request.url) + '/{localStorage}', response);
203203
response = response && HTTPSB.userSettings.deleteLocalStorage;
204204
if ( response ) {
205205
HTTPSB.localStorageRemovedCounter++;

‎js/httpsb.js‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ HTTPSB.normalizeScopeURL = function(url) {
3131
return null;
3232
}
3333
if ( url !== '*' ) {
34-
url = getRootURLFromURL(url);
34+
url = uriTools.rootURLFromURI(url);
3535
}
3636
return url;
3737
};
@@ -45,7 +45,7 @@ HTTPSB.createPageScopeIfNotExists = function(url) {
4545
if ( !url ) {
4646
return false;
4747
}
48-
url = getRootURLFromURL(url);
48+
url = uriTools.rootURLFromURI(url);
4949
var tscope = this.temporaryScopes.scopes[url];
5050
var pscope = this.permanentScopes.scopes[url];
5151
if ( !tscope !== !pscope ) {
@@ -88,7 +88,7 @@ HTTPSB.destroyPageScopeIfExists = function(url) {
8888
if ( !url || url === '*' ) {
8989
return false;
9090
}
91-
url = getRootURLFromURL(url);
91+
url = uriTools.rootURLFromURI(url);
9292
var tscope = this.temporaryScopes.scopes[url];
9393
var pscope = this.permanentScopes.scopes[url];
9494
if ( !tscope !== !pscope ) {
@@ -119,7 +119,7 @@ HTTPSB.scopePageExists = function(url) {
119119
if ( url === '*' ) {
120120
return true;
121121
}
122-
url = getRootURLFromURL(url);
122+
url = uriTools.rootURLFromURI(url);
123123
var tscope = this.temporaryScopes.scopes[url];
124124
var pscope = this.permanentScopes.scopes[url];
125125
if ( !tscope !== !pscope ) {

‎js/lists.js‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,8 @@ PermissionScope.prototype.evaluate = function(type, hostname) {
225225
var blacklist = this.black.list;
226226
var whitelist = this.white.list;
227227
var graylist = this.gray.list;
228-
var typeKey, cellKey, parent;
228+
var typeKey, cellKey;
229+
var parents, parent;
229230

230231
// Pick proper entry point
231232

@@ -243,6 +244,7 @@ PermissionScope.prototype.evaluate = function(type, hostname) {
243244
}
244245
// indirect: any type, specific hostname
245246
cellKey = '*|' + hostname;
247+
246248
// rhill 2013-10-26: Whitelist MUST be checked before blacklist,
247249
// because read-only blacklists are, hum... read-only? (which means
248250
// they can only be overriden through occultation, which means
@@ -261,9 +263,9 @@ PermissionScope.prototype.evaluate = function(type, hostname) {
261263
}
262264

263265
// indirect: parent hostname nodes
264-
parent = hostname;
266+
parents = uriTools.parentHostnamesFromHostname(hostname);
265267
while ( true ) {
266-
parent = getParentHostnameFromHostname(parent);
268+
parent = parents.shift();
267269
if ( !parent ) {
268270
break;
269271
}
@@ -277,12 +279,13 @@ PermissionScope.prototype.evaluate = function(type, hostname) {
277279
}
278280
// any type, specific parent
279281
cellKey = '*|' + parent;
282+
280283
// rhill 2013-10-26: Whitelist MUST be checked before blacklist,
281284
// because read-only blacklists are, hum... read-only?
282285
if ( whitelist[cellKey] ) {
283286
// https://github.com/gorhill/httpswitchboard/issues/29
284287
// The cell is indirectly whitelisted because of hostname, type
285-
// must nOT be blacklisted.
288+
// must NOT be blacklisted.
286289
if ( this.httpsb.userSettings.strictBlocking && blacklist[typeKey] ) {
287290
return 'rpt';
288291
}
@@ -315,9 +318,9 @@ PermissionScope.prototype.evaluate = function(type, hostname) {
315318
return 'rdt';
316319
}
317320
// indirect: parent hostname nodes
318-
parent = hostname;
321+
parents = uriTools.parentHostnamesFromHostname(hostname);
319322
while ( true ) {
320-
parent = getParentHostnameFromHostname(parent);
323+
parent = parents.shift();
321324
if ( !parent ) {
322325
break;
323326
}
@@ -482,7 +485,7 @@ PermissionScopes.prototype.normalizeScopeURL = function(url) {
482485
return null;
483486
}
484487
if ( url !== '*' ) {
485-
url = getRootURLFromURL(url);
488+
url = uriTools.rootURLFromURI(url);
486489
}
487490
return url;
488491
};
@@ -494,7 +497,7 @@ PermissionScopes.prototype.scopeFromURL = function(url) {
494497
return this.scopes['*'];
495498
}
496499
if ( url !== '*' ) {
497-
url = getRootURLFromURL(url);
500+
url = uriTools.rootURLFromURI(url);
498501
}
499502
return this.scopes[url];
500503
};

‎js/popup.js‎

Lines changed: 39 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -276,7 +276,7 @@ function initMatrixStats() {
276276
// collect all hostnames and ancestors from net traffic
277277
var background = getBackgroundPage();
278278
var pageUrl = pageStats.pageUrl;
279-
var url, hostname, reqType, root, parent, reqKey;
279+
var url, hostname, reqType, nodes, node, reqKey;
280280
var reqKeys = Object.keys(pageStats.requests);
281281
var iReqKeys = reqKeys.length;
282282

@@ -285,25 +285,31 @@ function initMatrixStats() {
285285
while ( iReqKeys-- ) {
286286
reqKey = reqKeys[iReqKeys];
287287
url = background.urlFromReqKey(reqKey);
288-
hostname = background.getHostnameFromURL(url);
288+
hostname = background.uriTools.hostnameFromURI(url);
289+
289290
// rhill 2013-10-23: hostname can be empty if the request is a data url
290291
// https://github.com/gorhill/httpswitchboard/issues/26
291292
if ( hostname === '' ) {
292-
hostname = background.getHostnameFromURL(pageUrl);
293+
hostname = background.uriTools.hostnameFromURI(pageUrl);
293294
}
294295
reqType = background.typeFromReqKey(reqKey);
296+
295297
// we want a row for self and ancestors
296-
parent = hostname;
297-
while ( parent ) {
298-
root = parent;
299-
if ( !matrixStats[parent] ) {
300-
matrixStats[parent] = HostnameStats.prototype.factory(parent);
298+
nodes = background.uriTools.allHostnamesFromHostname(hostname);
299+
300+
while ( true ) {
301+
node = nodes.shift();
302+
if ( !node ) {
303+
break;
304+
}
305+
if ( !matrixStats[node] ) {
306+
matrixStats[node] = HostnameStats.prototype.factory(node);
301307
}
302-
parent = background.getParentHostnameFromHostname(parent);
303308
}
304309
matrixStats[hostname].types[reqType].count += 1;
305310
// https://github.com/gorhill/httpswitchboard/issues/12
306311
// Count requests for whole row.
312+
307313
matrixStats[hostname].types['*'].count += 1;
308314
// meta row for domain, only:
309315
// - there are subdomains
@@ -358,8 +364,8 @@ function getGroupStats() {
358364
// First, group according to whether at least one node in the domain
359365
// hierarchy is white or blacklisted
360366
var background = getBackgroundPage();
361-
var pageDomain = background.getDomainFromURL(HTTPSBPopup.pageURL);
362-
var hostname, domain, parent;
367+
var pageDomain = background.uriTools.domainFromURI(HTTPSBPopup.pageURL);
368+
var hostname, domain, nodes, node;
363369
var temporaryColor;
364370
var dark, group;
365371
var hostnames = Object.keys(matrixStats);
@@ -377,17 +383,22 @@ function getGroupStats() {
377383
}
378384
// Walk upward the chain of hostname and find at least one which
379385
// is expressly whitelisted or blacklisted.
380-
parent = hostname;
381-
while ( parent ) {
382-
temporaryColor = matrixStats[parent].types['*'].temporaryColor;
386+
nodes = background.uriTools.allHostnamesFromHostname(hostname);
387+
domain = nodes[nodes.length-1];
388+
389+
while ( true ) {
390+
node = nodes.shift();
391+
if ( !node ) {
392+
break;
393+
}
394+
temporaryColor = matrixStats[node].types['*'].temporaryColor;
383395
dark = temporaryColor.charAt(1) === 'd';
384396
if ( dark ) {
385397
break;
386398
}
387-
parent = background.getParentHostnameFromHostname(parent);
388399
}
389400
// Domain of the page comes first
390-
if ( background.getDomainFromHostname(hostname) === pageDomain ) {
401+
if ( domain === pageDomain ) {
391402
group = 0;
392403
}
393404
// Whitelisted hostnames are second, blacklisted are fourth
@@ -397,7 +408,6 @@ function getGroupStats() {
397408
} else {
398409
group = 2;
399410
}
400-
domain = background.getDomainFromHostname(hostname);
401411
if ( !groups[group][domain] ) {
402412
groups[group][domain] = { all: {}, withRules: {} };
403413
}
@@ -410,6 +420,7 @@ function getGroupStats() {
410420
// which are not explicitly part of the web page.
411421
var iGroup = groups.length;
412422
var domains, iDomain;
423+
var nodes;
413424
while ( iGroup-- ) {
414425
group = groups[iGroup];
415426
domains = Object.keys(group);
@@ -419,10 +430,13 @@ function getGroupStats() {
419430
hostnames = Object.keys(group[domain].withRules);
420431
iHostname = hostnames.length;
421432
while ( iHostname-- ) {
422-
hostname = hostnames[iHostname];
423-
while ( hostname ) {
424-
group[domain].all[hostname] = group[domain].withRules[hostname];
425-
hostname = background.getParentHostnameFromHostname(hostname);
433+
nodes = background.uriTools.allHostnamesFromHostname(hostnames[iHostname]);
434+
while ( true ) {
435+
node = nodes.shift();
436+
if ( !node ) {
437+
break;
438+
}
439+
group[domain].all[node] = group[domain].withRules[node];
426440
}
427441
}
428442
}
@@ -717,22 +731,22 @@ function renderMatrixCellDomain(cell, domain) {
717731
$(cell).prop({reqType: '*', hostname: domain})
718732
.addClass(getCellClass(domain, '*'))
719733
.children('b')
720-
.text(domain);
734+
.text(punycode.toUnicode(domain));
721735
}
722736

723737
function renderMatrixCellSubdomain(cell, domain, subomain) {
724738
$(cell).prop({reqType: '*', hostname: subomain})
725739
.addClass(getCellClass(subomain, '*'))
726740
.children('b')
727-
.text(subomain.slice(0, subomain.lastIndexOf(domain)-1) + '.')
728-
.after(domain);
741+
.text(punycode.toUnicode(subomain.slice(0, subomain.lastIndexOf(domain)-1)) + '.')
742+
.after(punycode.toUnicode(domain));
729743
}
730744

731745
function renderMatrixMetaCellDomain(cell, domain) {
732746
$(cell).prop({reqType: '*', hostname: domain})
733747
.addClass(getCellClass(domain, '*'))
734748
.children('b')
735-
.text(domain)
749+
.text(punycode.toUnicode(domain))
736750
.before('\u2217.');
737751
}
738752

‎js/rulemanager.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ function renderRecipeStringToRule(recipe) {
8383
}
8484
// Validate hostname
8585
var hostname = parts[2];
86-
if ( hostname !== '*' && !getBackground().isValidHostname(hostname) ) {
86+
if ( hostname !== '*' && !getBackground().uriTools.isValidHostname(hostname) ) {
8787
return false;
8888
}
8989
// Validate type
@@ -139,7 +139,7 @@ function renderRecipeStringToScopeKey(recipe) {
139139
return false;
140140
}
141141
var scopeKey = parts[1];
142-
if ( scopeKey !== '*' && !getBackground().isValidRootURL(scopeKey) ) {
142+
if ( scopeKey !== '*' && !getBackground().uriTools.isValidRootURL(scopeKey) ) {
143143
return false;
144144
}
145145
return scopeKey;

‎js/start.js‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,8 @@ function injectedCodeCallback(r) {
5050
return;
5151
}
5252
var httpsb = HTTPSB;
53-
var pageUrl = normalizeChromiumUrl(r.pageUrl);
54-
var pageHostname = getHostnameFromURL(pageUrl);
53+
var pageUrl = uriTools.normalizeURI(r.pageUrl);
54+
var pageHostname = uriTools.hostnameFromURI(pageUrl);
5555
var sources, i;
5656
var url, domain, block;
5757
// scripts
@@ -64,8 +64,8 @@ function injectedCodeCallback(r) {
6464
domain = pageHostname;
6565
url = pageUrl + '{inline_script}';
6666
} else {
67-
url = normalizeChromiumUrl(url);
68-
domain = getHostnameFromURL(url);
67+
url = uriTools.normalizeURI(url);
68+
domain = uriTools.hostnameFromURI(url);
6969
}
7070
block = httpsb.blacklisted(pageUrl, 'script', domain);
7171
recordFromPageUrl(pageUrl, 'script', url, block);
@@ -75,8 +75,8 @@ function injectedCodeCallback(r) {
7575
sources = Object.keys(r.pluginSources);
7676
i = sources.length;
7777
while ( i-- ) {
78-
url = normalizeChromiumUrl(sources[i]);
79-
domain = getHostnameFromURL(url);
78+
url = uriTools.normalizeURI(sources[i]);
79+
domain = uriTools.hostnameFromURI(url);
8080
block = httpsb.blacklisted(pageUrl, 'object', domain);
8181
recordFromPageUrl(pageUrl, 'object', url, block);
8282
}
@@ -90,10 +90,10 @@ function onUpdatedTabsHandler(tabId, changeInfo, tab) {
9090
return;
9191
}
9292

93-
var pageUrl = normalizeChromiumUrl(tab.url);
93+
var pageUrl = uriTools.normalizeURI(tab.url);
9494

9595
// console.debug('tabs.onUpdated > tabId=%d changeInfo=%o tab=%o', tabId, changeInfo, tab);
96-
var protocol = getUrlProtocol(pageUrl);
96+
var protocol = uriTools.schemeFromURI(pageUrl);
9797
if ( protocol !== 'http' && protocol !== 'https' ) {
9898
return;
9999
}
@@ -159,7 +159,7 @@ function onBeforeNavigateCallback(details) {
159159
if ( HTTPSB.excludeRegex.test(details.url) ) {
160160
return;
161161
}
162-
var hostname = getHostnameFromURL(details.url);
162+
var hostname = uriTools.hostnameFromURI(details.url);
163163
setJavascript(hostname, HTTPSB.whitelisted(details.url, 'script', hostname));
164164
}
165165

@@ -194,7 +194,7 @@ load();
194194
var tab;
195195
while ( i-- ) {
196196
tab = tabs[i];
197-
bindTabToPageStats(tab.id, normalizeChromiumUrl(tab.url));
197+
bindTabToPageStats(tab.id, uriTools.normalizeURI(tab.url));
198198
}
199199
// Tabs are now bound to url stats stores, therefore it is now safe
200200
// to handle net traffic.

0 commit comments

Comments
 (0)