Merge pull request #2 from autogram-is/sets-and-filters

Sets and filters

Author: eaton

README.md

@@ -1,27 +1,9 @@
 # URL Tools
 
-Processing, normalizing, and de-duplicating large piles of URLs can be a pain,
-particularly if you're trying to distinguish "real" unique URLs from the many
-variations that can appear in the wild. URLs with anchor links, query params
-in different orders, social sharing and analytics campaign cruft, accidental
-references to staging servers… You get the idea.
+Processing, normalizing, and de-duplicating large piles of URLs can be a pain, particularly if you're trying to distinguish "real" unique URLs from the many variations that can appear in the wild. URLs with anchor links, query params in different orders, social sharing and analytics campaign cruft, accidental references to staging servers… You get the idea.
 
-URL Tools is a helper library whose sole purpose is making that process just a
-little less frustrating. It consists of four major pieces:
+URL Tools is a helper library whose sole purpose is making that process just a little less frustrating. It consists of four major pieces:
 
-- `ParsedUrl`, a wrapper for the standard WHATWG `URL` class that mixes in the
-  domain and subdomain parsing from `tldjs`. Serializing a `ParsedUrl` object to
-  JSON also produces a broken out collection of its individual properties, rather
-  than spitting out the `href` property, as is the `URL` class's habit.
-- `ParsedUrlSet`, a collection class that automatically parses, normalizes, and
-  de-duplicates sets of existing Urls. It's a bit janky, since ES6's `Set`
-  implementation only supports value comparison. As such, you can put `ParsedUrl`
-  objects *into* the set, but after normalization they're stored as simple strings.
-  It also keeps track of the URLs that it rejects as unparseable.
-- A light set of helper functions for common filtering and normalizing operations,
-  including sorting querystring parameters,  stripping social sharing cruft,
-  remapping 'ww1', 'ww2', etc. subdomains to a single canonical one, identifying
-  web vs. non-web URLs, flagging urls on public hosting like S3, and more.
-# Todo
-- [ ] A richer set of filters
-- [ ] Chainable filtering and transforming for ParsedUrlSet 
+- `ParsedUrl`, a wrapper for the standard WHATWG `URL` class that mixes in the domain and subdomain parsing from [`tldjs`](https://www.npmjs.com/package/tldjs). Serializing a `ParsedUrl` object to JSON also produces a broken out collection of its individual properties, rather than spitting out the `href` property, as is the `URL` class's habit.
+- `ParsedUrlSet`, a collection class that automatically parses, normalizes, and de-duplicates sets of existing Urls. It's a bit janky, since ES6's `Set` implementation only supports value comparison. As such, you can put `ParsedUrl` objects *into* the set, but after normalization they're stored as simple strings. It also keeps track of the URLs that it rejects as unparseable.
+- A light set of helper functions for common filtering and normalizing operations, including sorting querystring parameters,  stripping social sharing cruft, remapping 'ww1', 'ww2', etc. subdomains to a single canonical one, identifying web vs. non-web URLs, flagging urls on public hosting like S3, and more. These can be combined or defined on the fly to serve as the normalizer function for a `ParsedUrlSet`.

src/parsed-url-set.ts

@@ -1,9 +1,10 @@
 import {ParsedUrl} from './parsed-url';
 import {UrlMutator, Mutators} from './mutations';
+import {UrlFilter} from './filters';
 
 export class ParsedUrlSet extends Set<string> {
   static DefaultNormalizer: UrlMutator = Mutators.DefaultNormalizer;
-  readonly rejected: Set<string> = new Set<string>();
+  readonly unparseable: Set<string> = new Set<string>();
 
   public constructor(
     values: Array<string | ParsedUrl> = [],
@@ -18,7 +19,7 @@ export class ParsedUrlSet extends Set<string> {
     if (parsed) {
       super.add(parsed.href);
     } else {
-      this.rejected.add(value as string);
+      this.unparseable.add(value as string);
     }
     return this;
   }
@@ -36,7 +37,7 @@ export class ParsedUrlSet extends Set<string> {
   }
 
   override clear(): void {
-    this.rejected.clear();
+    this.unparseable.clear();
     super.clear();
   }
 
@@ -63,6 +64,12 @@ export class ParsedUrlSet extends Set<string> {
     }
   }
 
+  filter(filterFunction: UrlFilter): ParsedUrlSet {
+    let urls: ParsedUrl[] = [...this.hydrate()];
+    urls = urls.filter(u => filterFunction(u));
+    return new ParsedUrlSet(urls);
+  }
+
   hydrate(): ParsedUrl[] {
     return [...this].map(u => new ParsedUrl(u)) as ParsedUrl[];
   }

tests/parsed-url-set.ts

@@ -8,6 +8,24 @@ test('parsed url set normalizes duplicates', () => {
 
 test('parsed url ignores invalid URLs', () => {
   const set = new ParsedUrlSet(fixtures.UNPARSEABLE_URLS);
-  console.log([...set]);
   expect(set.size).toBe(0);
 });
+
+test('parsed url set is filterable', () => {
+  const set = new ParsedUrlSet([
+    fixtures.NORMALIZED_URL.normalized,
+    ...fixtures.NON_WEB_URLS,
+  ]);
+
+  expect(set.filter(u => u.domain === 'example.com').size).toBe(1);
+});
+
+test('parsed url set tracks unparseable rejections', () => {
+  const set = new ParsedUrlSet([
+    fixtures.NORMALIZED_URL.normalized,
+    ...fixtures.UNPARSEABLE_URLS,
+  ]);
+
+  expect(set.unparseable.size).toBe(fixtures.UNPARSEABLE_URLS.length);
+  expect(set.size).toBe(1);
+});