add more bots (#55)

Author: BenjaminAbt

src/MyCSharp.HttpUserAgentParser/HttpUserAgentStatics.cs

@@ -214,6 +214,21 @@ private static Regex CreateDefaultBrowserRegex(string key)
     public static readonly (string Key, string Value)[] Robots =
     [
         ( "googlebot", "Googlebot" ),
+        ( "meta-externalagent", "meta-externalagent" ),
+        ( "openai.com/searchbot", "OAI-SearchBot" ),
+        ( "CCBot", "CCBot" ),
+        ( "archive.org/details/archive.org_bot", "archive.org" ),
+        ( "opensiteexplorer.org/dotbot", "DotBot" ),
+        ( "awario.com/bots.html", "AwarioBot" ),
+        ( "Turnitin", "Turnitin" ),
+        ( "openai.com/gptbot", "GPTBot" ),
+        ( "perplexity.ai/perplexitybot", "PerplexityBot" ),
+        ( "developer.amazon.com/support/amazonbot", "Amazonbot" ),
+        ( "trendictionbot", "trendictionbot" ),
+        ( "openai.com/searchbot", "OAI-SearchBot" ),
+        ( "Bytespider", "Bytespider" ),
+        ( "MojeekBot", "MojeekBot" ),
+        ( "SeekportBot", "SeekportBot" ),
         ( "googleweblight", "Google Web Light" ),
         ( "PetalBot", "PetalBot"),
         ( "DuplexWeb-Google", "DuplexWeb-Google"),

tests/MyCSharp.HttpUserAgentParser.UnitTests/HttpUserAgentParserTests.cs

@@ -109,7 +109,6 @@ public void BrowserTests(string ua, string name, string version, string platform
     [InlineData("Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)", "MegaIndex")]
     [InlineData("Mozilla/5.0 (compatible; AhrefsBot/5.2; +http://ahrefs.com/robot/)", "Ahrefs")]
     [InlineData("Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)", "SEMRush")]
-    [InlineData("Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, [email protected])", "OpenSite")]
     [InlineData("Mozilla/5.0 (X11; U; Linux Core i7-4980HQ; de; rv:32.0; compatible; JobboerseBot; http://www.jobboerse.com/bot.htm) Gecko/20100101 Firefox/38.0", "Jobboerse")]
     [InlineData("Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)", "Majestic")]
     [InlineData("Mozilla/5.0 (compatible; SemrushBot/2~bl; +http://www.semrush.com/bot.html)", "SEMRush")]
@@ -132,6 +131,21 @@ public void BrowserTests(string ua, string name, string version, string platform
     [InlineData("WhatsApp/2.22.20.72 A", "WhatsApp")]
     [InlineData("WhatsApp/2.22.19.78 I", "WhatsApp")]
     [InlineData("WhatsApp/2.2236.3 N", "WhatsApp")]
+    [InlineData("Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amazonbot/0.1; +developer.amazon.com/support/amazonbot) Chrome/119.0.6045.214 Safari/537.36", "Amazonbot")]
+    [InlineData("Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +openai.com/gptbot)", "GPTBot")]
+    [InlineData("Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/116.0.1938.76 Safari/537.36", "BingBot")]
+    [InlineData("Mozilla/5.0 (compatible; AwarioBot/1.0; +awario.com/bots.html)", "AwarioBot")]
+    [InlineData("Mozilla/5.0 (compatible; DotBot/1.2; +opensiteexplorer.org/dotbot; [email protected])", "DotBot")]
+    [InlineData("Mozilla/5.0 (Windows NT 10.0; Win64; x64; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20100101 Firefox/125.0", "trendictionbot")]
+    [InlineData("Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; [email protected])", "Bytespider")]
+    [InlineData("Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +perplexity.ai/perplexitybot)", "PerplexityBot")]
+    [InlineData("Turnitin (bit.ly/2UvnfoQ)", "Turnitin")]
+    [InlineData("meta-externalagent/1.1 (+developers.facebook.com/docs/sharing/webmasters/crawler)", "meta-externalagent")]
+    [InlineData("CCBot/2.0 (commoncrawl.org/faq)", "CCBot")]
+    [InlineData("Mozilla/5.0 (compatible; SeekportBot; +bot.seekport.com)", "SeekportBot")]
+    [InlineData("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36; compatible; OAI-SearchBot/1.0; +openai.com/searchbot", "OAI-SearchBot")]
+    [InlineData("Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot)", "archive.org")]
+    [InlineData("Mozilla/5.0 (compatible; MojeekBot/0.11; +mojeek.com/bot.html)", "MojeekBot")]
     public void BotTests(string ua, string name)
     {
         HttpUserAgentInformation uaInfo = HttpUserAgentInformation.Parse(ua);

tests/MyCSharp.HttpUserAgentParser.UnitTests/HttpUserAgentPlatformInformationTests.cs

@@ -5,19 +5,22 @@
 
 namespace MyCSharp.HttpUserAgentParser.UnitTests;
 
-public class HttpUserAgentPlatformInformationTests
+public partial class HttpUserAgentPlatformInformationTests
 {
     [Theory]
     [InlineData("Batman", HttpUserAgentPlatformType.Android)]
     [InlineData("Robin", HttpUserAgentPlatformType.Windows)]
     public void Ctor(string name, HttpUserAgentPlatformType platform)
     {
-        Regex regex = new("");
+        Regex regex = EmptyRegex();
 
         HttpUserAgentPlatformInformation info = new(regex, name, platform);
 
         Assert.Equal(regex, info.Regex);
         Assert.Equal(name, info.Name);
         Assert.Equal(platform, info.PlatformType);
     }
+
+    [GeneratedRegex("")]
+    private static partial Regex EmptyRegex();
 }