Merge branch 'master' of https://github.com/CurseForgeCommunity/cflookup

Author: itssimple

CFLookup/CFLookup.csproj

@@ -14,6 +14,7 @@
 	  <PackageReference Include="Hangfire.AspNetCore" Version="1.8.22" />
 	  <PackageReference Include="Hangfire.Dashboard.BasicAuthorization" Version="1.0.2" />
 	  <PackageReference Include="Hangfire.Redis.StackExchange" Version="1.12.0" />
+	  <PackageReference Include="InfluxDB.Client" Version="4.18.0" />
 	  <PackageReference Include="Microsoft.AspNetCore.DataProtection.StackExchangeRedis" Version="9.0.11" />
 	  <PackageReference Include="Microsoft.Data.SqlClient" Version="5.1.5" />
 	  <PackageReference Include="Highsoft.Highcharts" Version="11.4.6.5" />

CFLookup/InfluxDBWriter.cs

@@ -0,0 +1,30 @@
+using CFLookup.Models;
+using InfluxDB.Client;
+using InfluxDB.Client.Api.Domain;
+using InfluxDB.Client.Writes;
+
+namespace CFLookup
+{
+    public class InfluxDBWriter(InfluxDBClient influxClient)
+    {
+        public async Task WriteBatchAsync(string org, string bucket, IEnumerable<InfluxProjectMetric> metrics)
+        {
+            var writeApi = influxClient.GetWriteApiAsync();
+
+            await writeApi.WritePointsAsync(
+                metrics.Select(m => 
+                    PointData.Measurement("cf_project_metrics")
+                        .Tag("project_id", m.ProjectId.ToString())
+                        .Tag("game_id", m.GameId.ToString())
+                        .Field("download_count", m.DownloadCount)
+                        .Field("thumbs_up_count", m.ThumbsUpCount)
+                        .Field("game_popularity_rank", m.GamePopularityRank)
+                        .Timestamp(m.Timestamp, WritePrecision.Ns)
+                ).ToList(),
+                bucket,
+                org,
+                CancellationToken.None
+            );
+        }
+    }
+}

CFLookup/Jobs/StoreCFApiProjects.cs

@@ -1,4 +1,5 @@
-using CurseForge.APIClient;
+using CFLookup.Models;
+using CurseForge.APIClient;
 using CurseForge.APIClient.Models.Mods;
 using Hangfire;
 using Hangfire.Server;
@@ -29,6 +30,8 @@ public async static Task RunAsync(PerformContext context, IJobCancellationToken
                     token.ThrowIfCancellationRequested();
 
                     var cfClient = scope.ServiceProvider.GetRequiredService<ApiClient>();
+                    
+                    var influxWriter = scope.ServiceProvider.GetRequiredService<InfluxDBWriter>();
 
                     var conn = scope.ServiceProvider.GetRequiredService<NpgsqlConnection>();
                     await conn.OpenAsync();
@@ -51,7 +54,7 @@ public async static Task RunAsync(PerformContext context, IJobCancellationToken
 
                         token.ThrowIfCancellationRequested();
 
-                        if (modList.Error != null && modList.Error.ErrorCode != 404)
+                        if (modList.Error != null && modList.Error.ErrorCode != 404 && !string.IsNullOrWhiteSpace(modList.Error.ErrorMessage))
                         {
                             // No-op for now, maybe Discord logs later
                             await SendDiscordErrorNotification(scope, $"The CF API threw an error at me: **{modList.Error.ErrorCode}**: {modList.Error.ErrorMessage}");
@@ -77,6 +80,10 @@ public async static Task RunAsync(PerformContext context, IJobCancellationToken
 
                         token.ThrowIfCancellationRequested();
 
+                        var projectMetrics = new List<InfluxProjectMetric>();
+                        
+                        var currentTs = DateTime.UtcNow;
+
                         foreach (var mod in modList.Data)
                         {
                             token.ThrowIfCancellationRequested();
@@ -237,13 +244,27 @@ ON CONFLICT (projectid, gameid) DO UPDATE
                             });
 
                             batch.BatchCommands.Add(cmd);
+                            
+                            projectMetrics.Add(new InfluxProjectMetric
+                            {
+                                ProjectId = mod.Id,
+                                GameId = mod.GameId,
+                                DownloadCount = mod.DownloadCount,
+                                ThumbsUpCount = mod.ThumbsUpCount,
+                                GamePopularityRank = mod.GamePopularityRank,
+                                Timestamp = currentTs
+                            });
 
                             if (batch.BatchCommands.Count >= 1000)
                             {
                                 if (!await ExecuteBatchWithRetries(batch))
                                 {
                                     // No-op for now, maybe Discord logs later
                                 }
+                                
+                                await influxWriter.WriteBatchAsync("CFLookup", "cf_project_data", projectMetrics);
+                                projectMetrics.Clear();
+                                currentTs = DateTime.UtcNow;
                                 token.ThrowIfCancellationRequested();
                             }
                         }
@@ -254,6 +275,10 @@ ON CONFLICT (projectid, gameid) DO UPDATE
                             {
                                 // No-op for now, maybe Discord logs later
                             }
+                            
+                            await influxWriter.WriteBatchAsync("CFLookup", "cf_project_data", projectMetrics);
+                            projectMetrics.Clear();
+                            
                             token.ThrowIfCancellationRequested();
                         }
 
@@ -330,4 +355,4 @@ private async static Task<bool> ExecuteBatchWithRetries(NpgsqlBatch batch, int r
             return false;
         }
     }
-}
+}

CFLookup/Models/InfluxProjectMetric.cs

@@ -0,0 +1,12 @@
+namespace CFLookup.Models
+{
+    public class InfluxProjectMetric
+    {
+        public required long ProjectId { get; set; }
+        public required int GameId { get; set; }
+        public required double DownloadCount { get; set; }
+        public required int ThumbsUpCount { get; set; }
+        public required int GamePopularityRank { get; set; }
+        public DateTime Timestamp { get; set; }
+    }
+}

CFLookup/Program.cs

@@ -1,4 +1,5 @@
 using CFLookup;
+using InfluxDB.Client;
 using Microsoft.AspNetCore.DataProtection;
 #if !DEBUG
 using CFLookup.Jobs;
@@ -60,6 +61,8 @@ private static async Task Main(string[] args)
         var hangfirePassword = string.Empty;
 
         var pgsqlConnString = string.Empty;
+        var influxDBConnString = string.Empty;
+        var influxToken = string.Empty;
 
         if (OperatingSystem.IsWindows())
         {
@@ -95,6 +98,16 @@ private static async Task Main(string[] args)
                               Environment.GetEnvironmentVariable("CFLOOKUP_PGSQL", EnvironmentVariableTarget.User) ??
                               Environment.GetEnvironmentVariable("CFLOOKUP_PGSQL", EnvironmentVariableTarget.Process) ??
                               string.Empty;
+            
+            influxDBConnString = Environment.GetEnvironmentVariable("CFLOOKUP_INFLUXDB", EnvironmentVariableTarget.Machine) ??
+                                 Environment.GetEnvironmentVariable("CFLOOKUP_INFLUXDB", EnvironmentVariableTarget.User) ??
+                                 Environment.GetEnvironmentVariable("CFLOOKUP_INFLUXDB", EnvironmentVariableTarget.Process) ??
+                                 string.Empty;
+            
+            influxToken = Environment.GetEnvironmentVariable("CFLOOKUP_INFLUXDB_TOKEN", EnvironmentVariableTarget.Machine) ??
+                            Environment.GetEnvironmentVariable("CFLOOKUP_INFLUXDB_TOKEN", EnvironmentVariableTarget.User) ??
+                            Environment.GetEnvironmentVariable("CFLOOKUP_INFLUXDB_TOKEN", EnvironmentVariableTarget.Process) ??
+                            string.Empty;
         }
         else
         {
@@ -104,6 +117,8 @@ private static async Task Main(string[] args)
             hangfireUser = Environment.GetEnvironmentVariable("CFLOOKUP_HangfireUser") ?? string.Empty;
             hangfirePassword = Environment.GetEnvironmentVariable("CFLOOKUP_HangfirePassword") ?? string.Empty;
             pgsqlConnString = Environment.GetEnvironmentVariable("CFLOOKUP_PGSQL") ?? string.Empty;
+            influxDBConnString = Environment.GetEnvironmentVariable("CFLOOKUP_INFLUXDB") ?? string.Empty;
+            influxToken = Environment.GetEnvironmentVariable("CFLOOKUP_INFLUXDB_TOKEN") ?? string.Empty;
         }
 
         var redis = ConnectionMultiplexer.Connect(redisServer);
@@ -112,6 +127,9 @@ private static async Task Main(string[] args)
 
         builder.Services.AddDataProtection()
             .PersistKeysToStackExchangeRedis(redis, "CFLookup-DataProtection-Keys");
+        
+        builder.Services.AddScoped(x => new InfluxDBClient(influxDBConnString, influxToken));
+        builder.Services.AddScoped<InfluxDBWriter>();
 
         builder.Services.AddScoped(x => new SqlConnection(dbConnectionString));