Add browser engine & family detection, improve test performance, update fixtures & regexes

.gitignore

@@ -1,3 +1,4 @@
+.DS_Store
 /.bundle/
 /.yardoc
 /Gemfile.lock
@@ -7,6 +8,7 @@
 /pkg/
 /spec/reports/
 /tmp/
+!tmp/.gitkeep
 *.bundle
 *.so
 *.o

.rspec

@@ -1 +1,3 @@
 --require spec_helper
+--format progress
+--format ParallelTests::RSpec::FailuresLogger --out tmp/failing_specs.log

.rubocop.yml

@@ -1,3 +1,6 @@
+plugins:
+  - rubocop-rspec
+
 AllCops:
   TargetRubyVersion: 3.4
   Exclude:

Gemfile

@@ -9,7 +9,11 @@ gem 'byebug'
 
 gem 'rake'
 gem 'rspec'
+gem 'parallel_tests'
 gem 'rubocop', '>= 1.75'
+gem 'rubocop-rspec'
+gem 'bootsnap'
+gem 'oj'
 
 gem 'browser', require: false
 gem 'useragent', require: false

Rakefile

@@ -2,10 +2,14 @@
 
 require 'rake'
 require 'rake/testtask'
+require 'rspec/core/rake_task'
+require 'parallel_tests'
 
 $LOAD_PATH.unshift 'lib'
 require 'device_detector'
 
+RSpec::Core::RakeTask.new(:spec)
+
 desc 'generate detectable names output for README'
 task :detectable_names do
   require 'date'
@@ -42,14 +46,14 @@ task :detectable_names do
 end
 
 MATOMO_REPO_URL = 'https://github.com/matomo-org/device-detector'
-MATOMO_REPO_TAG = '6.4.6'
+MATOMO_COMMIT_SHA = '1b521fb382873602ea5d3fed582a7c9d72afbd6f'
 MATOMO_CHECKOUT_LOCATION = '/tmp/matomo_device_detector'
 
 def matomo_checkout!
   if File.exist?(MATOMO_CHECKOUT_LOCATION)
-    system "cd #{MATOMO_CHECKOUT_LOCATION}; git fetch origin; git reset --hard #{MATOMO_REPO_TAG}"
+    system "cd #{MATOMO_CHECKOUT_LOCATION}; git fetch origin; git reset --hard #{MATOMO_COMMIT_SHA}"
   else
-    system "git clone --depth 100 #{MATOMO_REPO_URL} -b #{MATOMO_REPO_TAG} #{MATOMO_CHECKOUT_LOCATION}"
+    system "git clone --depth 100 #{MATOMO_REPO_URL} --revision=#{MATOMO_COMMIT_SHA} #{MATOMO_CHECKOUT_LOCATION}"
   end
 end
 

lib/device_detector.rb

@@ -4,7 +4,6 @@
 
 require 'device_detector/version'
 require 'device_detector/memory_cache'
-
 require 'device_detector/client_hint'
 
 require 'device_detector/parser/abstract_parser'
@@ -17,6 +16,8 @@
 require 'device_detector/parser/client/media_player'
 require 'device_detector/parser/client/pim'
 require 'device_detector/parser/client/browser'
+require 'device_detector/parser/client/browser_module/engine'
+require 'device_detector/parser/client/browser_module/engine/version'
 require 'device_detector/parser/client/library'
 require 'device_detector/parser/device/abstract_device_parser'
 require 'device_detector/parser/device/hbb_tv'
@@ -37,6 +38,38 @@ class DeviceDetector
   REGEX_CACHE = ::DeviceDetector::MemoryCache.new({})
   private_constant :REGEX_CACHE
 
+  class << self
+    @@parser_classes = [
+      Parser::Client::FeedReader,
+      Parser::Client::MobileApp,
+      Parser::Client::MediaPlayer,
+      Parser::Client::Pim,
+      Parser::Client::Browser,
+      Parser::Client::Library,
+      Parser::Device::HbbTv,
+      Parser::Device::ShellTv,
+      Parser::Device::Notebook,
+      Parser::Device::Console,
+      Parser::Device::CarBrowser,
+      Parser::Device::Camera,
+      Parser::Device::PortableMediaPlayer,
+      Parser::Device::Mobile,
+      Parser::Bot
+    ]
+
+    def root
+      @root ||= File.expand_path('..', __dir__)
+    end
+
+    def regexes_dir
+      @regexes_dir ||= File.join(root, 'regexes')
+    end
+
+    def parser_classes
+      @@parser_classes
+    end
+  end
+
   attr_reader :client_hint, :user_agent
 
   def initialize(user_agent = nil, headers = nil)
@@ -45,23 +78,9 @@ def initialize(user_agent = nil, headers = nil)
     @vendor_fragment_parser = DeviceDetector::Parser::VendorFragment.new
     @operating_system_parser = DeviceDetector::Parser::OperatingSystem.new
 
-    add_parser(Parser::Client::FeedReader.new)
-    add_parser(Parser::Client::MobileApp.new)
-    add_parser(Parser::Client::MediaPlayer.new)
-    add_parser(Parser::Client::Pim.new)
-    add_parser(Parser::Client::Browser.new)
-    add_parser(Parser::Client::Library.new)
-
-    add_parser(Parser::Device::HbbTv.new)
-    add_parser(Parser::Device::ShellTv.new)
-    add_parser(Parser::Device::Notebook.new)
-    add_parser(Parser::Device::Console.new)
-    add_parser(Parser::Device::CarBrowser.new)
-    add_parser(Parser::Device::Camera.new)
-    add_parser(Parser::Device::PortableMediaPlayer.new)
-    add_parser(Parser::Device::Mobile.new)
-
-    add_parser(Parser::Bot.new)
+    self.class.parser_classes.each do |klass|
+      add_parser(klass.new)
+    end
 
     use(user_agent, headers) if user_agent || headers
   end
@@ -126,9 +145,7 @@ class Configuration
       attr_accessor :max_cache_keys
 
       def to_hash
-        {
-          max_cache_keys: max_cache_keys
-        }
+        { max_cache_keys: max_cache_keys }
       end
     end
 
@@ -140,6 +157,10 @@ def cache
       @cache ||= MemoryCache.new(config.to_hash)
     end
 
+    def reset_cache!
+      @cache = MemoryCache.new(config.to_hash)
+    end
+
     def configure
       @config = Configuration.new
       yield(config)
@@ -430,9 +451,7 @@ def should_parse?
   end
 
   def presence(var)
-    return nil if var.nil?
-    return nil if var.empty?
-    return nil if var == ''
+    return nil if var.to_s.empty?
 
     var
   end

lib/device_detector/memory_cache.rb

@@ -39,6 +39,10 @@ def get_or_set(key, value = nil)
       set(string_key, value)
     end
 
+    def purge!
+      @data = {}
+    end
+
     private
 
     def get_hit(key)

lib/device_detector/parser/abstract_parser.rb

@@ -3,11 +3,29 @@
 class DeviceDetector
   module Parser
     class AbstractParser
-      # overriden
-      def self.client_hint_mapping
-        {}
+      class << self
+        # overriden
+        def client_hint_mapping
+          {}
+        end
+
+        def add_fixture_path(path)
+          @custom_fixture_paths = (custom_fixture_paths << path).uniq
+          REGEX_CACHE.purge!
+        end
+
+        def custom_fixture_paths
+          @custom_fixture_paths ||= []
+        end
+
+        def reset_custom_fixtures!
+          @custom_fixture_paths = []
+          REGEX_CACHE.purge!
+        end
       end
 
+      attr_writer :user_agent, :client_hints
+
       REGEX_CACHE = ::DeviceDetector::MemoryCache.new({})
       private_constant :REGEX_CACHE
 
@@ -21,15 +39,10 @@ def use(uas, hints)
         @client_hints = hints
       end
 
-      attr_writer :user_agent, :client_hints
-
       protected
 
       def empty?(var)
-        return true if var.nil?
-        return true if var.empty?
-
-        false
+        var.to_s.empty?
       end
 
       def fuzzy_compare(val1, val2)
@@ -40,9 +53,7 @@ def build_version(version_string, matches)
         return unless version_string
 
         version_string = build_by_match(version_string, matches)
-        version_string = version_string.gsub('_', '.')
-
-        version_string.strip.sub(/^(\.+)/, '').sub(/(\.+)$/, '')
+        version_string.gsub('_', '.').chomp('.')
       end
 
       def build_by_match(item, matches)
@@ -107,6 +118,10 @@ def fixture_file
         ''
       end
 
+      def fixture_paths
+        ([File.join(DeviceDetector.regexes_dir, fixture_file)] + self.class.custom_fixture_paths).uniq.compact
+      end
+
       def parser_name
         ''
       end
@@ -125,10 +140,29 @@ def regexes
         end
       end
 
+      def load_regex_file(path)
+        YAML.safe_load_file(path, permitted_classes: [String, Integer, NilClass, Array, Hash])
+      rescue Errno::ENOENT
+        warn "[#{self.class}] Fixture file not found: #{path}"
+        nil
+      end
+
       def load_regexes
         REGEX_CACHE.get_or_set(fixture_file) do
-          YAML.safe_load_file(fixture_file,
-                              permitted_classes: [String, Integer, NilClass, Array, Hash])
+          result = nil
+          fixture_paths.each do |fixture_path|
+            next unless File.file?(fixture_path)
+
+            result =  case result
+                      when Array
+                        result + load_regex_file(fixture_path)
+                      when Hash
+                        result.merge(load_regex_file(fixture_path))
+                      else
+                        load_regex_file(fixture_path) || result
+                      end
+          end
+          result
         end
       end
 
@@ -202,9 +236,9 @@ def prepare_definition_for_cache(definition)
         definition
       end
 
-      def regex_from_user_agent_cache(key = nil, &block)
+      def regex_from_user_agent_cache(key = nil, &)
         key = "#{parser_name}_#{@user_agent}#{key}"
-        DeviceDetector.cache.get_or_set(key, &block)
+        DeviceDetector.cache.get_or_set(key, &)
       end
 
       def deep_symbolize_keys(obj)
@@ -220,6 +254,13 @@ def deep_symbolize_keys(obj)
           obj
         end
       end
+
+      def satisfied_by_version?(requirement_string, version)
+        requirement = Gem::Requirement.new(requirement_string)
+        requirement.satisfied_by?(Gem::Version.new(version))
+      rescue Gem::Requirement::BadRequirementError
+        true
+      end
     end
   end
 end

lib/device_detector/parser/bot.rb

@@ -18,7 +18,7 @@ def parse
       protected
 
       def fixture_file
-        'regexes/bots.yml'
+        'bots.yml'
       end
 
       def parser_name