Increase lookup performance by ~426x

Added a configurable internal memory cache that makes subsequent lookups basically instantaneous. The lookup benchmarks between 1.4.5 and 1.4.6 dropped from 9801 msec to 28 msec when recalling 7778000 filters.

Flattened duplicate constructor code.

Author: TechnikEmpire

DistillNET/DistillNET/DistillNET.csproj

@@ -8,14 +8,13 @@
     <PackageProjectUrl>https://github.com/TechnikEmpire/DistillNET</PackageProjectUrl>
     <RepositoryUrl>https://github.com/TechnikEmpire/DistillNET</RepositoryUrl>
     <Description>DistillNET is a library for matching and filtering HTTP requests URLs using the Adblock Plus Filter format.</Description>
-    <Copyright>Copyright © 2017 Jesse Nicholson</Copyright>
-    <Version>1.4.5</Version>
+    <Copyright>Copyright © 2017 - 2018 Jesse Nicholson</Copyright>
+    <Version>1.4.6</Version>
     <Authors>Jesse Nicholson</Authors>
     <Company>Technik Empire</Company>
     <PackageTags>DistillNET Adblock AdblockPlus Adblock-Plus URL-Filter URL-Filtering Content-Filter Filter</PackageTags>
-    <PackageReleaseNotes>Fixes a bug where, when using the FilterDbCollection constructor that takes parameters, if you specified to use an in-memory database, the database was not configured correctly.
-
-Package also now bundles XML documentation.</PackageReleaseNotes>
+    <PackageReleaseNotes>Improves rule lookup performance by a factor of ~426!</PackageReleaseNotes>
+    <AssemblyVersion>1.4.6.0</AssemblyVersion>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
@@ -27,7 +26,8 @@ Package also now bundles XML documentation.</PackageReleaseNotes>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Data.SQLite" Version="2.0.0" />
+    <PackageReference Include="Microsoft.Data.SQLite" Version="2.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="2.0.1" />
   </ItemGroup>
 
 </Project>

DistillNET/DistillNET/DistillNET/FilterDbCollection.cs

@@ -7,12 +7,15 @@
 
 using DistillNET.Extensions;
 using Microsoft.Data.Sqlite;
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Options;
 using System;
 using System.Collections.Generic;
 using System.Data;
 using System.IO;
 using System.Linq;
 using System.Reflection;
+using System.Threading;
 using System.Threading.Tasks;
 
 namespace DistillNET
@@ -42,33 +45,23 @@ public class FilterDbCollection : IDisposable
         private readonly string m_globalKey;
 
         /// <summary>
-        /// Constructs a new FilterDbCollection using an in-memory database.
+        /// Memory cache.
         /// </summary>
-        public FilterDbCollection()
-        {
-            
-            var version = typeof(FilterDbCollection).Assembly.GetCustomAttribute<AssemblyFileVersionAttribute>().Version;
-            var rnd = new Random();
-            var rndNum = rnd.Next();
-            var generatedDbName = string.Format("{0} {1} - {2}", nameof(FilterDbCollection), version, rndNum);
-
-            // "Data Source = :memory:; Cache = shared;"
-            var cb = new SqliteConnectionStringBuilder();
-            cb.DataSource = generatedDbName;
-            cb.Mode = SqliteOpenMode.Memory;
-            cb.Cache = SqliteCacheMode.Shared;            
-            m_connection = new SqliteConnection(cb.ToString());
-
-            //m_connection. = SQLiteConnectionFlags.UseConnectionPool | SQLiteConnectionFlags.NoConvertSettings | SQLiteConnectionFlags.NoVerifyTypeAffinity;            
-            m_connection.Open();
-
-            ConfigureDatabase();
-
-            CreateTables();
+        private MemoryCache m_cache;
 
-            m_globalKey = "global";
+        /// <summary>
+        /// Mem cache options.
+        /// </summary>
+        private MemoryCacheOptions m_cacheOptions;
 
-            m_ruleParser = new AbpFormatRuleParser();
+        /// <summary>
+        /// Constructs a new FilterDbCollection using an in-memory database.
+        /// </summary>
+        /// <param name="cacheOptions">
+        /// User defined query caching options.
+        /// </param>
+        public FilterDbCollection(MemoryCacheOptions cacheOptions = null) : this(null, true, true, cacheOptions)
+        {   
         }
 
         /// <summary>
@@ -84,13 +77,24 @@ public FilterDbCollection()
         /// <param name="useMemory">
         /// If true, the database will be created as a purely in-memory database.
         /// </param>
-        public FilterDbCollection(string dbAbsolutePath, bool overwrite = true, bool useMemory = false)
+        /// <param name="cacheOptions">
+        /// User defined query caching options.
+        /// </param>
+        public FilterDbCollection(string dbAbsolutePath, bool overwrite = true, bool useMemory = false, MemoryCacheOptions cacheOptions = null)
         {
             if(!useMemory && overwrite && File.Exists(dbAbsolutePath))
             {
                 File.Delete(dbAbsolutePath);
             }
 
+            if(cacheOptions == null)
+            {
+                cacheOptions = new MemoryCacheOptions();                
+                cacheOptions.ExpirationScanFrequency = TimeSpan.FromMinutes(10);
+            }
+
+            m_cacheOptions = cacheOptions;
+
             bool isNew = !File.Exists(dbAbsolutePath);
 
             m_ruleParser = new AbpFormatRuleParser();
@@ -213,6 +217,16 @@ public void FinalizeForRead()
             CreatedIndexes();
         }
 
+        private void RecreateCache()
+        {
+            if (m_cache != null)
+            {   
+                m_cache.Dispose();
+            }
+
+            m_cache = new MemoryCache(m_cacheOptions);
+        }
+
         /// <summary>
         /// Parses the supplied list of rules and stores them in the assigned database for retrieval,
         /// indexed by the rule's domain names.
@@ -230,6 +244,8 @@ public void FinalizeForRead()
         /// </returns>
         public async Task<Tuple<int, int>> ParseStoreRules(string[] rawRuleStrings, short categoryId)
         {
+            RecreateCache();
+
             int loaded = 0, failed = 0;
 
             using(var transaction = m_connection.BeginTransaction())
@@ -305,6 +321,8 @@ public async Task<Tuple<int, int>> ParseStoreRules(string[] rawRuleStrings, shor
         /// </returns>
         public async Task<Tuple<int, int>> ParseStoreRulesFromStream(Stream rawRulesStream, short categoryId)
         {
+            RecreateCache();
+
             int loaded = 0, failed = 0;
 
             using(var transaction = m_connection.BeginTransaction())
@@ -408,7 +426,16 @@ public async Task<List<UrlFilter>> GetWhitelistFiltersForDomain(string domain =
         /// </returns>
         private async Task<List<UrlFilter>> GetFiltersForDomain(string domain, bool isWhitelist)
         {
-            var retVal = new List<UrlFilter>();
+            var cacheKey = new Tuple<string, bool>(domain, isWhitelist);
+
+            List<UrlFilter> retVal;
+
+            if(m_cache.TryGetValue(cacheKey, out retVal))
+            {
+                return retVal;
+            }
+
+            retVal = new List<UrlFilter>();
 
             var allPossibleVariations = GetAllPossibleSubdomains(domain);
 
@@ -454,6 +481,8 @@ private async Task<List<UrlFilter>> GetFiltersForDomain(string domain, bool isWh
                 }
             }
 
+            m_cache.Set(cacheKey, retVal);
+
             return retVal;
         }
 

DistillNET/Tests/Program.cs

@@ -55,7 +55,7 @@ private static void TestDomainWideException()
         }
 
         private static void Main(string[] args)
-        {
+        {   
             var parser = new AbpFormatRuleParser();
 
             string easylistPath = AppDomain.CurrentDomain.BaseDirectory + "easylist.txt";

DistillNET/Tests/Tests.csproj

@@ -65,13 +65,16 @@
     </Content>
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="DistillNET">
-      <Version>1.4.5</Version>
-    </PackageReference>
     <PackageReference Include="Microsoft.Data.SQLite">
-      <Version>2.0.0</Version>
+      <Version>2.0.1</Version>
     </PackageReference>
   </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\DistillNET\DistillNET.csproj">
+      <Project>{693840ca-eaf1-4718-bb1f-e2f9d6646e10}</Project>
+      <Name>DistillNET</Name>
+    </ProjectReference>
+  </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <PropertyGroup>
     <PostBuildEvent>