#1964 The ability to create a stripped-down file with user agents is added.

It ll save resources on next running the nopCommerce

Author: skoshelev

.gitignore

@@ -162,4 +162,5 @@ src/Presentation/Nop.Web/Plugins/*
 src/Presentation/Nop.Web/Content/Images/Thumbs/*
 src/Presentation/Nop.Web/App_Data/InstalledPlugins.txt
 src/Presentation/Nop.Web/App_Data/Settings.txt
+src/Presentation/Nop.Web/App_Data/browscap.crawlersonly.xml
 src/Presentation/Nop.Web/Administration/db_backups/*.bak

src/Libraries/Nop.Core/Configuration/NopConfig.cs

@@ -29,7 +29,8 @@ public object Create(object parent, object configContext, XmlNode section)
 
             var userAgentStringsNode = section.SelectSingleNode("UserAgentStrings");
             config.UserAgentStringsPath = GetString(userAgentStringsNode, "databasePath");
-           
+            config.CrawlerOnlyUserAgentStringsPath = GetString(userAgentStringsNode, "crawlersOnlyDatabasePath");
+
             var supportPreviousNopcommerceVersionsNode = section.SelectSingleNode("SupportPreviousNopcommerceVersions");
             config.SupportPreviousNopcommerceVersions = GetBool(supportPreviousNopcommerceVersionsNode, "Enabled");
 
@@ -79,6 +80,11 @@ private T SetByXElement<T>(XmlNode node, string attrName, Func<string, T> conver
         /// </summary>
         public string UserAgentStringsPath { get; private set; }
 
+        /// <summary>
+        /// Path to database with crawler only user agent strings
+        /// </summary>
+        public string CrawlerOnlyUserAgentStringsPath { get; private set; }
+
 
 
         /// <summary>

src/Libraries/Nop.Services/Helpers/BrowscapXmlParser.cs

@@ -16,30 +16,67 @@ public class BrowscapXmlHelper
     {
         private readonly List<string> _crawlerUserAgentsRegexp;
 
-        public BrowscapXmlHelper(string filePath)
+        public BrowscapXmlHelper(string userAgentStringsPath, string crawlerOnlyUserAgentStringsPath)
         {
             _crawlerUserAgentsRegexp = new List<string>();
 
-            Initialize(filePath);
+            Initialize(userAgentStringsPath, crawlerOnlyUserAgentStringsPath);
         }
 
-        private void Initialize(string filePath)
+        private void Initialize(string userAgentStringsPath, string crawlerOnlyUserAgentStringsPath)
         {
-            using (var sr = new StreamReader(filePath))
+            List<XElement> crawlerItems = null;
+
+            if (!string.IsNullOrEmpty(crawlerOnlyUserAgentStringsPath) && File.Exists(crawlerOnlyUserAgentStringsPath))
+            {
+                //try to load crawler list from crawlers only file
+                using (var sr = new StreamReader(crawlerOnlyUserAgentStringsPath))
+                {
+                    crawlerItems = XDocument.Load(sr).Root.Return(x => x.Elements("browscapitem").ToList(), null);
+                }
+            }
+
+            if (crawlerItems == null)
+            {
+                //try to load crawler list from full user agents file
+                using (var sr = new StreamReader(userAgentStringsPath))
+                {
+                    crawlerItems = XDocument.Load(sr).Root.Return(x => x.Element("browsercapitems"), null)
+                        //only crawlers
+                        .Return(x => x.Elements("browscapitem").Where(IsBrowscapItemIsCrawler).ToList(), null);
+                }
+            }
+
+            if (crawlerItems == null)
+                throw new Exception("Incorrect file format");
+
+            _crawlerUserAgentsRegexp.AddRange(crawlerItems
+                //get only user agent names
+                .Select(e => e.Attribute("name"))
+                .Where(e => e != null && !string.IsNullOrEmpty(e.Value))
+                .Select(e => e.Value)
+                .Select(ToRegexp));
+
+            if (string.IsNullOrEmpty(crawlerOnlyUserAgentStringsPath) || File.Exists(crawlerOnlyUserAgentStringsPath))
+                return;
+
+            //try to write crawlers file
+            using (var sw = new StreamWriter(crawlerOnlyUserAgentStringsPath))
             {
-                var browsercapItems = XDocument.Load(sr).Root.Return(x => x.Element("browsercapitems"), null);
+                var root = new XElement("browsercapitems");
 
-                if (browsercapItems == null)
-                    throw new Exception("Incorrect file format");
+                foreach (var crawler in crawlerItems)
+                {
+                    foreach (var element in crawler.Elements().ToList())
+                    {
+                        if (element.Attribute("name").Return(x => x.Value.ToLower(), string.Empty) == "crawler")
+                            continue;
+                        element.Remove();
+                    }
 
-                _crawlerUserAgentsRegexp.AddRange(browsercapItems.Elements("browscapitem")
-                    //only crawlers
-                    .Where(IsBrowscapItemIsCrawler)
-                    //get only user agent names
-                    .Select(e => e.Attribute("name"))
-                    .Where(e => e != null && !string.IsNullOrEmpty(e.Value))
-                    .Select(e => e.Value)
-                    .Select(ToRegexp));
+                    root.Add(crawler);
+                }
+                root.Save(sw);
             }
         }
 

src/Libraries/Nop.Services/Helpers/UserAgentHelper.cs

@@ -45,8 +45,10 @@ protected virtual BrowscapXmlHelper GetBrowscapXmlHelper()
                 if (Singleton<BrowscapXmlHelper>.Instance != null)
                     return Singleton<BrowscapXmlHelper>.Instance;
 
-                var filePath = CommonHelper.MapPath(_config.UserAgentStringsPath);
-                var browscapXmlHelper = new BrowscapXmlHelper(filePath);
+                var userAgentStringsPath = CommonHelper.MapPath(_config.UserAgentStringsPath);
+                var crawlerOnlyUserAgentStringsPath = string.IsNullOrEmpty(_config.CrawlerOnlyUserAgentStringsPath) ? string.Empty : CommonHelper.MapPath(_config.CrawlerOnlyUserAgentStringsPath);
+
+                var browscapXmlHelper = new BrowscapXmlHelper(userAgentStringsPath, crawlerOnlyUserAgentStringsPath);
                 Singleton<BrowscapXmlHelper>.Instance = browscapXmlHelper;
 
                 return Singleton<BrowscapXmlHelper>.Instance;

src/Presentation/Nop.Web/Nop.Web.csproj

@@ -1281,6 +1281,7 @@
 		App_Data\InstalledPlugins.txt;
 		App_Data\Settings.txt;
 		App_Data\Nop.Db.sdf;
+        App_Data\browscap.crawlersonly.xml;
 		bin\Nop.Admin.dll.config;
 		bin\Nop.Web.dll.config;
 		Content\Files\ExportImport\*.txt;

src/Presentation/Nop.Web/Web.config

@@ -29,7 +29,8 @@
     <!-- Redis support (used by web farms, Azure, etc). Find more about it at https://azure.microsoft.com/en-us/documentation/articles/cache-dotnet-how-to-use-azure-redis-cache/ -->
     <RedisCaching Enabled="false" ConnectionString="localhost" />
     <!-- You can get the latest version of user agent strings at http://browscap.org/ -->
-    <UserAgentStrings databasePath="~/App_Data/browscap.xml" />
+    <!-- Leave "crawlersOnlyDatabasePath" attribute empty if you want to use full version of "browscap.xml" file -->
+    <UserAgentStrings databasePath="~/App_Data/browscap.xml" crawlersOnlyDatabasePath="~/App_Data/browscap.crawlersonly.xml"/>
     <!-- Set the setting below to "False" if you did not upgrade from one of the previous versions. It can slightly improve performance -->
     <SupportPreviousNopcommerceVersions Enabled="True" />
     <!-- Do not edit this element. For advanced users only -->