Merge pull request #94 from maxmind/horgh/normalize

Replace fewer TLDs when normalizing

Author: ugexe

CHANGELOG.md

@@ -17,11 +17,8 @@
 * Duplicate `.com`s are now removed from email domain names when
   `hash_address` is used. For example, `example.com.com` will become
   `example.com`.
-* Extraneous characters after `.com` are now removed from email domain
-  names when `hash_address` is used. For example, `example.comfoo` will
-  become `example.com`.
-* Certain `.com` typos are now normalized to `.com` when `hash_address` is
-  used. For example, `example.cam` will become `example.com`.
+* Certain TLD typos are now normalized when `hash_address` is used. For
+  example, `example.comcom` will become `example.com`.
 * Additional `gmail.com` domain names with leading digits are now
   normalized when `hash_address` is used. For example, `100gmail.com` will
   become `gmail.com`.

lib/minfraud/components/email.rb

@@ -131,6 +131,55 @@ def clean_email_address(address)
       }.freeze
       private_constant :TYPO_DOMAINS
 
+      TYPO_TLDS = {
+        'comm'   => 'com',
+        'commm'  => 'com',
+        'commmm' => 'com',
+        'comn'   => 'com',
+
+        'cbm'    => 'com',
+        'ccm'    => 'com',
+        'cdm'    => 'com',
+        'cem'    => 'com',
+        'cfm'    => 'com',
+        'cgm'    => 'com',
+        'chm'    => 'com',
+        'cim'    => 'com',
+        'cjm'    => 'com',
+        'ckm'    => 'com',
+        'clm'    => 'com',
+        'cmm'    => 'com',
+        'cnm'    => 'com',
+        'cpm'    => 'com',
+        'cqm'    => 'com',
+        'crm'    => 'com',
+        'csm'    => 'com',
+        'ctm'    => 'com',
+        'cum'    => 'com',
+        'cvm'    => 'com',
+        'cwm'    => 'com',
+        'cxm'    => 'com',
+        'cym'    => 'com',
+        'czm'    => 'com',
+
+        'col'    => 'com',
+        'con'    => 'com',
+
+        'dom'    => 'com',
+        'don'    => 'com',
+        'som'    => 'com',
+        'son'    => 'com',
+        'vom'    => 'com',
+        'von'    => 'com',
+        'xom'    => 'com',
+        'xon'    => 'com',
+
+        'clam'   => 'com',
+        'colm'   => 'com',
+        'comcom' => 'com',
+      }.freeze
+      private_constant :TYPO_TLDS
+
       EQUIVALENT_DOMAINS = {
         'googlemail.com' => 'gmail.com',
         'pm.me'          => 'protonmail.com',
@@ -330,10 +379,16 @@ def clean_domain(domain)
         domain = SimpleIDN.to_ascii(domain)
 
         domain.sub!(/(?:\.com){2,}$/, '.com')
-        domain.sub!(/\.com[^.]+$/, '.com')
-        domain.sub!(/(?:\.(?:com|c[a-z]{1,2}m|co[ln]|[dsvx]o[mn]|))$/, '.com')
         domain.sub!(/^\d+(?:gmail?\.com)$/, 'gmail.com')
 
+        idx = domain.rindex('.')
+        if !idx.nil?
+          tld = domain[idx + 1..]
+          if TYPO_TLDS.key?(tld)
+            domain = "#{domain[0, idx]}.#{TYPO_TLDS[tld]}"
+          end
+        end
+
         if TYPO_DOMAINS.key?(domain)
           domain = TYPO_DOMAINS[domain]
         end

spec/components/email_spec.rb

@@ -63,7 +63,7 @@
         { input: '  [email protected]', output: '[email protected]' },
         {
           input:  '[email protected]|abc124472372',
-          output: '[email protected]',
+          output: '[email protected]|abc124472372',
         },
         { input: '[email protected]', output: '[email protected]' },
         { input: '[email protected]', output: '[email protected]' },
@@ -76,9 +76,10 @@
         { input: '[email protected]', output: '[email protected]' },
         { input: '[email protected]', output: '[email protected]' },
         { input: '[email protected]', output: '[email protected]' },
-        { input: '[email protected]', output: 'foo@example.com' },
-        { input: '[email protected]', output: 'foo@example.com' },
+        { input: '[email protected]', output: 'foo@example.comfoo' },
+        { input: '[email protected]', output: 'foo@example.cam' },
         { input: '[email protected]', output: '[email protected]' },
+        { input: '[email protected]', output: '[email protected]' },
       ]
 
       tests.each do |i|