Slightly refactor to allow adding of referers by method instead of file

Author: Kelley Reynolds

ruby/README.md

@@ -46,6 +46,10 @@ parser = RefererParser::Parser.new('http://example.com/path/to/other/referers.ym
 parser = RefererParser::Parser.new
 parser.update('/path/to/internal.yml')
 
+# Default referers, then add your own internal domain inline instead of from a file
+parser = RefererParser::Parser.new
+parser.add_referer('internal', 'SnowPlow', 'snowplowanalytics.com')
+
 # Clear all of the existing referers
 parser.clear!
 ```

ruby/lib/referer-parser/parser.rb

@@ -22,6 +22,9 @@ class Parser
 
     # Create a new parser from one or more filenames/uris, defaults to ../data/referers.json
     def initialize(uris=DefaultFile)
+      @domain_index ||= {}
+      @name_hash ||= {}
+
       update(uris)
     end
 
@@ -37,10 +40,48 @@ def update(uris)
     # Clean out the database
     def clear!
       @domain_index, @name_hash = {}, {}
-      
+
       true
     end
 
+    # Add a referer to the database with medium, name, domain or array of domains, and a parameter or array of parameters
+    # If called manually and a domain is added to an existing entry with a path, you may need to call optimize_index! afterwards.
+    def add_referer(medium, name, domains, parameters=nil)
+      # The same name can be used with multiple mediums so we make a key here
+      name_key = "#{name}-#{medium}"
+
+      # Update the name has with the parameter and medium data
+      @name_hash[name_key] = {:source => name, :medium => medium, :parameters => [parameters].flatten }
+
+      # Update the domain to name index
+      [domains].flatten.each do |domain_url|
+        domain, *path = domain_url.split('/')
+        if domain =~ /\Awww\.(.*)\z/i
+          domain = $1
+        end
+
+        domain.downcase!
+
+        @domain_index[domain] ||= []
+        if !path.empty?
+          @domain_index[domain] << ['/' + path.join('/'), name_key]
+        else
+          @domain_index[domain] << ['/', name_key]
+        end
+      end
+    end
+
+    # Prune duplicate entries and sort with the most specific path first if there is more than one entry
+    # In this case, sorting by the longest string works fine
+    def optimize_index!
+      @domain_index.each do |key, val|
+        # Sort each path/name_key pair by the longest path
+        @domain_index[key].sort! { |a, b|
+          b[0].size <=> a[0].size
+        }.uniq!
+      end
+    end
+
     # Given a string or URI, return a hash of data
     def parse(obj)
       url = obj.is_a?(URI) ? obj : URI.parse(obj.to_s)
@@ -141,7 +182,7 @@ def deserialize_json(data)
 
     def read_referer_data(uri)
       # Attempt to read the data from the network if application, or the file on the local system
-      if uri =~ /^(?:ht|f)tps?:\/\//
+      if uri =~ /\A(?:ht|f)tps?:\/\//
         require 'open-uri'
         begin
           open(uri).read
@@ -160,44 +201,13 @@ def read_referer_data(uri)
     # Format of the name_hash:
     #   { name_key => {:source, :medium, :parameters} }
     def parse_referer_data(data)
-      @domain_index ||= {}
-      @name_hash ||= {}
-
       data.each do |medium, name_hash|
         name_hash.each do |name, name_data|
-          # The same name can be used with multiple mediums so we make a key here
-          name_key = "#{name}-#{medium}"
-
-          # Update the name has with the parameter and medium data
-          @name_hash[name_key] = {:source => name, :medium => medium, :parameters => name_data['parameters'] }
-
-          # Update the domain to name index
-          name_data['domains'].each do |domain_url|
-            domain, *path = domain_url.split('/')
-            if domain =~ /\Awww\.(.*)\z/i
-              domain = $1
-            end
-
-            domain.downcase!
-
-            @domain_index[domain] ||= []
-            if !path.empty?
-              @domain_index[domain] << ['/' + path.join('/'), name_key]
-            else
-              @domain_index[domain] << ['/', name_key]
-            end
-          end
+          add_referer(medium, name, name_data['domains'], name_data['parameters'])
         end
       end
 
-      # Prune duplicate entries and sort with the most specific path first if there is more than one entry
-      # In this case, sorting by the longest string works fine
-      @domain_index.each do |key, val|
-        # Sort each path/name_key pair by the longest path
-        @domain_index[key].sort! { |a, b|
-          b[0].size <=> a[0].size
-        }.uniq!
-      end
+      optimize_index!
     rescue
       raise CorruptReferersError.new("Unable to parse referer data", $!)
     end

ruby/spec/parser_spec.rb

@@ -21,79 +21,89 @@
   let(:internal_parser) { RefererParser::Parser.new(fixture('internal.json')) }
   let(:combined_parser) { RefererParser::Parser.new([RefererParser::Parser::DefaultFile, fixture('internal.json')]) }
   let(:remote_parser) { RefererParser::Parser.new(remote_file) }
+  let(:domain_index) { parser.instance_variable_get(:@domain_index) }
+  let(:name_hash) { parser.instance_variable_get(:@name_hash) }
+
+  # This gets overridden for different parsers in subsections
+  let(:parser) { default_parser }
 
   describe "exceptions" do
     it "should raise UnsupportedFormatError" do
-      lambda { default_parser.update(__FILE__) }.should raise_error(RefererParser::UnsupportedFormatError)
+      lambda { parser.update(__FILE__) }.should raise_error(RefererParser::UnsupportedFormatError)
     end
 
     it "should raise CorruptReferersError with invalid json" do
-      lambda { default_parser.update(fixture('invalid.json')) }.should raise_error(RefererParser::CorruptReferersError)
+      lambda { parser.update(fixture('invalid.json')) }.should raise_error(RefererParser::CorruptReferersError)
     end
 
     it "should raise CorruptReferersError with invalid yaml" do
-      lambda { default_parser.update(fixture('invalid.yml')) }.should raise_error(RefererParser::CorruptReferersError)
+      lambda { parser.update(fixture('invalid.yml')) }.should raise_error(RefererParser::CorruptReferersError)
     end
 
     it "should raise CorruptReferersError with valid file with invalid data" do
-      lambda { default_parser.update(fixture('referer-tests.json')) }.should raise_error(RefererParser::CorruptReferersError)
+      lambda { parser.update(fixture('referer-tests.json')) }.should raise_error(RefererParser::CorruptReferersError)
     end
 
     it "should raise InvalidUriError with insane" do
-      lambda { default_parser.parse('>total gibberish<') }.should raise_error(RefererParser::InvalidUriError)
+      lambda { parser.parse('>total gibberish<') }.should raise_error(RefererParser::InvalidUriError)
     end
 
     it "should raise InvalidUriError with non http(s)" do
-      lambda { default_parser.parse('ftp://ftp.really.com/whatever.json') }.should raise_error(RefererParser::InvalidUriError)
+      lambda { parser.parse('ftp://ftp.really.com/whatever.json') }.should raise_error(RefererParser::InvalidUriError)
     end
   end
 
   describe "with the default parser" do
     it "should have a non-empty domain_index" do
-      default_parser.instance_variable_get(:@domain_index).should_not be_empty
+      domain_index.should_not be_empty
     end
 
     it "should have a non-empty name_hash" do
-      default_parser.instance_variable_get(:@name_hash).should_not be_empty
+      name_hash.should_not be_empty
     end
 
     it "should be clearable" do
-      default_parser.clear!
-      default_parser.instance_variable_get(:@name_hash).should be_empty
-      default_parser.instance_variable_get(:@domain_index).should be_empty
+      parser.clear!
+      name_hash.should be_empty
+      domain_index.should be_empty
     end
 
     it "should be updatable" do
-      size = default_parser.instance_variable_get(:@domain_index).size
-      default_parser.update(fixture('internal.json'))
-      default_parser.instance_variable_get(:@domain_index).size.should > size
+      size = domain_index.size
+      parser.update(fixture('internal.json'))
+      domain_index.size.should > size
     end
   end
 
   describe "with the internal parser" do
+    let(:parser) { internal_parser }
+
     it "should have internal mediums only" do
-      internal_parser.instance_variable_get(:@domain_index).each_value do |(arr)|
+      domain_index.each_value do |(arr)|
         path, name_key = arr[0], arr[1]
-        internal_parser.instance_variable_get(:@name_hash)[name_key][:medium].should == 'internal'
+        name_hash[name_key][:medium].should == 'internal'
       end
     end
   end
 
   describe "with the remote parser" do
+    let(:parser) { remote_parser }
+
     # These are combined here to reduce network fetches
     it "should have a non-empty domain_index and name_hash" do
-      remote_parser.instance_variable_get(:@domain_index).should_not be_empty
-      remote_parser.instance_variable_get(:@name_hash).should_not be_empty
+      domain_index.should_not be_empty
+      name_hash.should_not be_empty
     end
   end
 
   describe "sample fixtures" do
+    let(:parser) { combined_parser }
     # Include our internal data as well
     JSON.parse(File.read(File.join(File.dirname(__FILE__), 'fixtures', 'referer-tests.json'))).each do |fixture|
       it fixture['spec'] do
         parsed_as_string, parsed_as_uri = nil, nil
-        lambda { parsed_as_string = combined_parser.parse(fixture['uri']) }.should_not raise_error
-        lambda { parsed_as_uri = combined_parser.parse(URI.parse(fixture['uri'])) }.should_not raise_error
+        lambda { parsed_as_string = parser.parse(fixture['uri']) }.should_not raise_error
+        lambda { parsed_as_uri = parser.parse(URI.parse(fixture['uri'])) }.should_not raise_error
 
         ['source', 'term', 'known', 'medium'].each do |key|
           parsed_as_uri[key.to_sym].should == fixture[key]
@@ -105,18 +115,67 @@
 
   describe "general behavior" do
     it "should return the better result when the referer contains two or more parameters" do
-      parsed = default_parser.parse("http://search.tiscali.it/?tiscalitype=web&collection=web&q=&key=hello")
+      parsed = parser.parse("http://search.tiscali.it/?tiscalitype=web&collection=web&q=&key=hello")
       parsed[:term].should == "hello"
     end
 
     it "should return the better result when the referer contains same parameters" do
-      parsed = default_parser.parse("http://search.tiscali.it/?tiscalitype=web&collection=web&key=&key=hello")
+      parsed = parser.parse("http://search.tiscali.it/?tiscalitype=web&collection=web&key=&key=hello")
       parsed[:term].should == "hello"
     end
 
     it "should return the normalized domain" do
-      parsed = default_parser.parse("http://it.images.search.YAHOO.COM/images/view;_ylt=A0PDodgQmGBQpn4AWQgdDQx.;_ylu=X3oDMTBlMTQ4cGxyBHNlYwNzcgRzbGsDaW1n?back=http%3A%2F%2Fit.images.search.yahoo.com%2Fsearch%2Fimages%3Fp%3DEarth%2BMagic%2BOracle%2BCards%26fr%3Dmcafee%26fr2%3Dpiv-web%26tab%3Dorganic%26ri%3D5&w=1064&h=1551&imgurl=mdm.pbzstatic.com%2Foracles%2Fearth-magic-oracle-cards%2Fcard-1.png&rurl=http%3A%2F%2Fwww.psychicbazaar.com%2Foracles%2F143-earth-magic-oracle-cards.html&size=2.8+KB&name=Earth+Magic+Oracle+Cards+-+Psychic+Bazaar&p=Earth+Magic+Oracle+Cards&oid=f0a5ad5c4211efe1c07515f56cf5a78e&fr2=piv-web&fr=mcafee&tt=Earth%2BMagic%2BOracle%2BCards%2B-%2BPsychic%2BBazaar&b=0&ni=90&no=5&ts=&tab=organic&sigr=126n355ib&sigb=13hbudmkc&sigi=11ta8f0gd&.crumb=IZBOU1c0UHU")
+      parsed = parser.parse("http://it.images.search.YAHOO.COM/images/view;_ylt=A0PDodgQmGBQpn4AWQgdDQx.;_ylu=X3oDMTBlMTQ4cGxyBHNlYwNzcgRzbGsDaW1n?back=http%3A%2F%2Fit.images.search.yahoo.com%2Fsearch%2Fimages%3Fp%3DEarth%2BMagic%2BOracle%2BCards%26fr%3Dmcafee%26fr2%3Dpiv-web%26tab%3Dorganic%26ri%3D5&w=1064&h=1551&imgurl=mdm.pbzstatic.com%2Foracles%2Fearth-magic-oracle-cards%2Fcard-1.png&rurl=http%3A%2F%2Fwww.psychicbazaar.com%2Foracles%2F143-earth-magic-oracle-cards.html&size=2.8+KB&name=Earth+Magic+Oracle+Cards+-+Psychic+Bazaar&p=Earth+Magic+Oracle+Cards&oid=f0a5ad5c4211efe1c07515f56cf5a78e&fr2=piv-web&fr=mcafee&tt=Earth%2BMagic%2BOracle%2BCards%2B-%2BPsychic%2BBazaar&b=0&ni=90&no=5&ts=&tab=organic&sigr=126n355ib&sigb=13hbudmkc&sigi=11ta8f0gd&.crumb=IZBOU1c0UHU")
       parsed[:domain].should == "images.search.yahoo.com"
     end
   end
+
+  describe "optimize_index" do
+    let(:domains) { ['fnord.com', 'fnord.com', 'fnord.com/path'] }
+
+    before do
+      parser.add_referer('internal', 'Fnord', domains)
+    end
+
+    it "should have out of order and duplicate domains before optimization" do
+      domain_index['fnord.com'].transpose.first.should == ['/', '/', '/path']
+    end
+
+    it "should have out of order domains before optimization" do
+      parser.optimize_index!
+      domain_index['fnord.com'].transpose.first.should == ['/path', '/']
+    end
+  end
+
+  describe "add_referer" do
+    it "should add a referer to the domain_index" do
+      domain_index['fnord.com'].should be_nil
+      parser.add_referer('internal', 'Fnord', 'fnord.com')
+      domain_index['fnord.com'].should_not be_nil
+    end
+
+    it "should add a referer with multiple domains to the domain_index" do
+      domain_index['fnord.com'].should be_nil
+      domain_index['boo.com'].should be_nil
+      parser.add_referer('internal', 'Fnord', ['fnord.com', 'boo.com'])
+      domain_index['fnord.com'].should_not be_nil
+      domain_index['boo.com'].should_not be_nil
+    end
+
+    it "should add a referer to the name_hash" do
+      name_hash['fnord.com-internal'].should be_nil
+      parser.add_referer('internal', 'Fnord', 'fnord.com')
+      name_hash['Fnord-internal'].should_not be_nil
+    end
+
+    it "should add parameters to the name_hash" do
+      parser.add_referer('internal', 'Fnord', 'fnord.com', ['Q', 'q'])
+      name_hash['Fnord-internal'][:parameters].should == ['Q', 'q']
+    end
+
+    it "should add a single parameter to the name_hash" do
+      parser.add_referer('internal', 'Fnord', 'fnord.com', 'q')
+      name_hash['Fnord-internal'][:parameters].should == ['q']
+    end
+  end
 end