int to long

Author: Sunny-DotNet

src/CommonCrawl/Handlers/StatisticsHandler.cs

@@ -26,7 +26,8 @@ public async Task<StatisticsInfo> GetStatisticsAsync(string collectionId,
         await foreach (var line in lines)
         {
             // 解析每一行数据并填充到result中
-            var parts = line.Split('\t');
+            //BUG:CC-MAIN-2016-36 has a few lines that does not start with '['
+            var parts = (line.StartsWith('[') ? line : $"[{line}").Split('\t');
             if (parts.Length != 2) continue;
             var args1 = JsonSerializer.Deserialize<JsonElement[]>(parts[0]);
             var type = args1![0].GetString();
@@ -41,7 +42,7 @@ public async Task<StatisticsInfo> GetStatisticsAsync(string collectionId,
                     {
 
                         var counts = parts[1].StartsWith('[')
-                            ? JsonSerializer.Deserialize<int[]>(parts[1])!
+                            ? JsonSerializer.Deserialize<long[]>(parts[1])!
                             :
                             [
                                 int.Parse(parts[1]),
@@ -73,8 +74,8 @@ public async Task<StatisticsInfo> GetStatisticsAsync(string collectionId,
                 case "tld":
                     {
                         var counts = parts[1].StartsWith('[')
-                            ? JsonSerializer.Deserialize<int[]>(parts[1])!
-                            : [int.Parse(parts[1])];
+                            ? JsonSerializer.Deserialize<long[]>(parts[1])!
+                            : [long.Parse(parts[1])];
                         var domain = args1![1]!.GetString()!;
                     }
 

src/CommonCrawl/Models/StatisticsInfo.cs

@@ -2,36 +2,36 @@
 {
     public class StatisticsInfo
     {
-        public Dictionary<int, int> HttpStatusCodes { get; } = new();
-        public Dictionary<string, int> PageCharsets { get; } = new();
-        public Dictionary<string, int> UrlCharsets { get; } = new();
-        public Dictionary<string, int> PageLanguages { get; } = new();
-        public Dictionary<string, int> UrlLanguages { get; } = new();
+        public Dictionary<int, long> HttpStatusCodes { get; } = new();
+        public Dictionary<string, long> PageCharsets { get; } = new();
+        public Dictionary<string, long> UrlCharsets { get; } = new();
+        public Dictionary<string, long> PageLanguages { get; } = new();
+        public Dictionary<string, long> UrlLanguages { get; } = new();
         public long SurtDomainTotal { set; get; }
         public long DomainTotal { set; get; }
         public long HostTotal { set; get; }
         public long FetchTotal { set; get; }
         public long PageTotal { set; get; }
         public long UrlTotal { set; get; }
-        public void AddHttpStatusCode(int statusCode, int result)
+        public void AddHttpStatusCode(int statusCode, long result)
         {
             HttpStatusCodes[statusCode] = result;
         }
 
-        public void AddPageCharset(string charset, int count)
+        public void AddPageCharset(string charset, long count)
         {
             PageCharsets[charset] = count;
         }
-        public void AddUrlCharset(string charset, int count)
+        public void AddUrlCharset(string charset, long count)
         {
             UrlCharsets[charset] = count;
         }
 
-        public void AddPageLanguage(string language, int count)
+        public void AddPageLanguage(string language, long count)
         {
             PageLanguages[language] = count;
         }
-        public void AddUrlLanguage(string language, int count)
+        public void AddUrlLanguage(string language, long count)
         {
             UrlLanguages[language] = count;
         }