Skip to content

Commit f63fc15

Browse files
itssimpleitssimple
committed
feat: Store CF project info
1 parent f095a35 commit f63fc15

File tree

4 files changed

+325
-4
lines changed

4 files changed

+325
-4
lines changed

‎CFLookup/CFLookup.csproj‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,15 @@
1111
</PropertyGroup>
1212

1313
<ItemGroup>
14-
<PackageReference Include="Hangfire.AspNetCore" Version="1.8.18" />
14+
<PackageReference Include="Hangfire.AspNetCore" Version="1.8.21" />
1515
<PackageReference Include="Hangfire.Dashboard.BasicAuthorization" Version="1.0.2" />
1616
<PackageReference Include="Hangfire.Redis.StackExchange" Version="1.12.0" />
1717
<PackageReference Include="Microsoft.Data.SqlClient" Version="5.1.5" />
1818
<PackageReference Include="Highsoft.Highcharts" Version="11.4.6.5" />
1919
<PackageReference Include="Highsoft.Highstock" Version="11.4.6.5" />
20-
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
21-
<PackageReference Include="NSec.Cryptography" Version="24.4.0" />
20+
<PackageReference Include="Newtonsoft.Json" Version="13.0.4" />
21+
<PackageReference Include="Npgsql" Version="9.0.4" />
22+
<PackageReference Include="NSec.Cryptography" Version="25.4.0" />
2223
<PackageReference Include="CurseForge.APIClient" Version="4.2.0" />
2324
<PackageReference Include="StackExchange.Redis" Version="2.8.31" />
2425
</ItemGroup>

‎CFLookup/Jobs/StoreCFApiProjects.cs‎

Lines changed: 290 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,290 @@
1+
using CurseForge.APIClient;
2+
using CurseForge.APIClient.Models.Mods;
3+
using Hangfire;
4+
using Hangfire.Server;
5+
using Npgsql;
6+
using System.Text.Json;
7+
8+
namespace CFLookup.Jobs
9+
{
10+
[AutomaticRetry(Attempts = 0)]
11+
public class StoreCFApiProjects
12+
{
13+
const int BUCKET_SIZE = 10_000;
14+
private const int EMPTY_BUCKETS = 25;
15+
private const int RETRY_BATCH = 3;
16+
17+
public async static Task RunAsync(PerformContext context)
18+
{
19+
using (var scope = Program.ServiceProvider.CreateScope())
20+
{
21+
var cfClient = scope.ServiceProvider.GetRequiredService<ApiClient>();
22+
cfClient.RequestDelay = TimeSpan.FromSeconds(0.05);
23+
cfClient.RequestTimeout = TimeSpan.FromSeconds(30);
24+
25+
var conn = scope.ServiceProvider.GetRequiredService<NpgsqlConnection>();
26+
await conn.OpenAsync();
27+
28+
var emptyBuckets = 0;
29+
30+
var buckets = GetBucketRanges(1, int.MaxValue);
31+
32+
foreach (var bucket in buckets)
33+
{
34+
var _bucket = Enumerable.Range(bucket.start, bucket.items);
35+
36+
var modList = await cfClient.GetModsByIdListAsync(new GetModsByIdsListRequestBody
37+
{
38+
FilterPcOnly = true,
39+
ModIds = _bucket.ToList()
40+
});
41+
42+
if (modList.Error != null && modList.Error.ErrorCode != 404)
43+
{
44+
// No-op for now, maybe Discord logs later
45+
}
46+
47+
if (modList.Data.Count == 0)
48+
{
49+
emptyBuckets++;
50+
if (emptyBuckets >= EMPTY_BUCKETS)
51+
{
52+
break;
53+
}
54+
}
55+
else
56+
{
57+
emptyBuckets = 0;
58+
}
59+
60+
await using var tx = await conn.BeginTransactionAsync();
61+
await using var batch = new NpgsqlBatch(conn);
62+
batch.Transaction = tx;
63+
batch.Timeout = 600;
64+
65+
foreach (var mod in modList.Data)
66+
{
67+
var cmd = new NpgsqlBatchCommand("""
68+
69+
INSERT INTO project_data (
70+
projectid,
71+
gameid,
72+
name,
73+
slug,
74+
links,
75+
summary,
76+
status,
77+
downloadcount,
78+
isfeatured,
79+
primarycategoryid,
80+
categories,
81+
classid,
82+
authors,
83+
logo,
84+
screenshots,
85+
mainfileid,
86+
latestfiles,
87+
latestfileindexes,
88+
datecreated,
89+
datemodified,
90+
datereleased,
91+
allowmoddistribution,
92+
gamepopularityrank,
93+
isavailable,
94+
thumbsupcount,
95+
rating
96+
)
97+
VALUES (
98+
$1,
99+
$2,
100+
$3,
101+
$4,
102+
$5,
103+
$6,
104+
$7,
105+
$8,
106+
$9,
107+
$10,
108+
$11,
109+
$12,
110+
$13,
111+
$14,
112+
$15,
113+
$16,
114+
$17,
115+
$18,
116+
$19,
117+
$20,
118+
$21,
119+
$22,
120+
$23,
121+
$24,
122+
$25,
123+
$26
124+
)
125+
ON CONFLICT (projectid, gameid) DO UPDATE
126+
SET
127+
name=EXCLUDED.name,
128+
slug=EXCLUDED.slug,
129+
links=EXCLUDED.links,
130+
summary=EXCLUDED.summary,
131+
status=EXCLUDED.status,
132+
downloadcount=EXCLUDED.downloadcount,
133+
isfeatured=EXCLUDED.isfeatured,
134+
primarycategoryid=EXCLUDED.primarycategoryid,
135+
categories=EXCLUDED.categories,
136+
classid=EXCLUDED.classid,
137+
authors=EXCLUDED.authors,
138+
logo=EXCLUDED.logo,
139+
screenshots=EXCLUDED.screenshots,
140+
mainfileid=EXCLUDED.mainfileid,
141+
latestfiles=EXCLUDED.latestfiles,
142+
latestfileindexes=EXCLUDED.latestfileindexes,
143+
datecreated=EXCLUDED.datecreated,
144+
datemodified=EXCLUDED.datemodified,
145+
datereleased=EXCLUDED.datereleased,
146+
allowmoddistribution=EXCLUDED.allowmoddistribution,
147+
gamepopularityrank=EXCLUDED.gamepopularityrank,
148+
isavailable=EXCLUDED.isavailable,
149+
thumbsupcount=EXCLUDED.thumbsupcount,
150+
rating=EXCLUDED.rating,
151+
latestupdate=timezone('UTC'::text, now());
152+
153+
""");
154+
155+
cmd.Parameters.AddWithValue(mod.Id);
156+
cmd.Parameters.AddWithValue(mod.GameId);
157+
cmd.Parameters.AddWithValue(mod.Name);
158+
cmd.Parameters.AddWithValue(mod.Slug);
159+
cmd.Parameters.Add(new NpgsqlParameter()
160+
{
161+
NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Jsonb, Value = JsonSerializer.Serialize(mod.Links)
162+
});
163+
cmd.Parameters.AddWithValue(mod.Summary.Replace("\0", ""));
164+
cmd.Parameters.AddWithValue((int)mod.Status);
165+
cmd.Parameters.AddWithValue(mod.DownloadCount);
166+
cmd.Parameters.AddWithValue(mod.IsFeatured);
167+
cmd.Parameters.AddWithValue(mod.PrimaryCategoryId);
168+
cmd.Parameters.Add(new NpgsqlParameter
169+
{
170+
NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Jsonb,
171+
Value = JsonSerializer.Serialize(mod.Categories)
172+
});
173+
cmd.Parameters.Add(new NpgsqlParameter
174+
{
175+
NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Integer,
176+
Value = (object?)mod.ClassId ?? DBNull.Value
177+
});
178+
cmd.Parameters.Add(new NpgsqlParameter
179+
{
180+
NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Jsonb, Value = JsonSerializer.Serialize(mod.Authors)
181+
});
182+
cmd.Parameters.Add(new NpgsqlParameter
183+
{
184+
NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Jsonb, Value = JsonSerializer.Serialize(mod.Logo)
185+
});
186+
cmd.Parameters.Add(new NpgsqlParameter
187+
{
188+
NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Jsonb,
189+
Value = JsonSerializer.Serialize(mod.Screenshots)
190+
});
191+
cmd.Parameters.AddWithValue(mod.MainFileId);
192+
cmd.Parameters.Add(new NpgsqlParameter
193+
{
194+
NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Jsonb,
195+
Value = JsonSerializer.Serialize(mod.LatestFiles)
196+
});
197+
cmd.Parameters.Add(new NpgsqlParameter
198+
{
199+
NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Jsonb,
200+
Value = JsonSerializer.Serialize(mod.LatestFilesIndexes)
201+
});
202+
cmd.Parameters.AddWithValue(mod.DateCreated);
203+
cmd.Parameters.AddWithValue(mod.DateModified);
204+
cmd.Parameters.AddWithValue(mod.DateReleased);
205+
cmd.Parameters.Add(new NpgsqlParameter
206+
{
207+
NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Boolean,
208+
Value = (object?)mod.AllowModDistribution ?? DBNull.Value
209+
});
210+
cmd.Parameters.AddWithValue(mod.GamePopularityRank);
211+
cmd.Parameters.AddWithValue(mod.IsAvailable);
212+
cmd.Parameters.AddWithValue(mod.ThumbsUpCount);
213+
cmd.Parameters.Add(new NpgsqlParameter
214+
{
215+
NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Double, Value = (object?)mod.Rating ?? DBNull.Value
216+
});
217+
218+
batch.BatchCommands.Add(cmd);
219+
220+
if (batch.BatchCommands.Count >= 1000)
221+
{
222+
if (!await ExecuteBatchWithRetries(batch))
223+
{
224+
// No-op for now, maybe Discord logs later
225+
}
226+
}
227+
}
228+
229+
if (batch.BatchCommands.Count > 0)
230+
{
231+
if (!await ExecuteBatchWithRetries(batch))
232+
{
233+
// No-op for now, maybe Discord logs later
234+
}
235+
}
236+
237+
await tx.CommitAsync();
238+
}
239+
}
240+
241+
BackgroundJob.Schedule(() => StoreCFApiProjects.RunAsync(null), TimeSpan.FromMinutes(30));
242+
}
243+
244+
private async static Task<bool> ExecuteBatchWithRetries(NpgsqlBatch batch, int retries = RETRY_BATCH)
245+
{
246+
for (var attempt = 1; attempt <= retries; attempt++)
247+
{
248+
try
249+
{
250+
await batch.ExecuteNonQueryAsync();
251+
batch.BatchCommands.Clear();
252+
return true;
253+
}
254+
catch (Exception)
255+
{
256+
if (attempt == retries)
257+
{
258+
return false;
259+
}
260+
261+
await Task.Delay(1000 * attempt);
262+
}
263+
}
264+
265+
return false;
266+
}
267+
268+
private static List<(int start, int end, int items)> GetBucketRanges(int start, int max)
269+
{
270+
var buckets = new List<(int start, int end, int items)>();
271+
272+
while (start <= max)
273+
{
274+
var bucketSize = Math.Min(BUCKET_SIZE, max - start + 1);
275+
var bucket = Enumerable.Range(start, bucketSize);
276+
277+
buckets.Add((bucket.First(), bucket.Last(), bucket.Count()));
278+
start += bucketSize;
279+
280+
if (start < 0)
281+
{
282+
// We went around, stop it
283+
break;
284+
}
285+
}
286+
287+
return buckets;
288+
}
289+
}
290+
}

‎CFLookup/Program.cs‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,13 @@
22
#if !DEBUG
33
using CFLookup.Jobs;
44
using Hangfire;
5+
using Hangfire.Storage;
56
using Hangfire.Dashboard.BasicAuthorization;
67
using Hangfire.Redis.StackExchange;
78
#endif
89
using Microsoft.AspNetCore.ResponseCompression;
910
using Microsoft.Data.SqlClient;
11+
using Npgsql;
1012
using StackExchange.Redis;
1113
using System.IO.Compression;
1214

@@ -50,6 +52,8 @@ private static async Task Main(string[] args)
5052

5153
var hangfireUser = string.Empty;
5254
var hangfirePassword = string.Empty;
55+
56+
var pgsqlConnString = string.Empty;
5357

5458
if (OperatingSystem.IsWindows())
5559
{
@@ -77,6 +81,11 @@ private static async Task Main(string[] args)
7781
Environment.GetEnvironmentVariable("CFLOOKUP_HangfirePassword", EnvironmentVariableTarget.User) ??
7882
Environment.GetEnvironmentVariable("CFLOOKUP_HangfirePassword", EnvironmentVariableTarget.Process) ??
7983
string.Empty;
84+
85+
pgsqlConnString = Environment.GetEnvironmentVariable("CFLOOKUP_PGSQL", EnvironmentVariableTarget.Machine) ??
86+
Environment.GetEnvironmentVariable("CFLOOKUP_PGSQL", EnvironmentVariableTarget.User) ??
87+
Environment.GetEnvironmentVariable("CFLOOKUP_PGSQL", EnvironmentVariableTarget.Process) ??
88+
string.Empty;
8089
}
8190
else
8291
{
@@ -85,13 +94,16 @@ private static async Task Main(string[] args)
8594
dbConnectionString = Environment.GetEnvironmentVariable("CFLOOKUP_SQL") ?? string.Empty;
8695
hangfireUser = Environment.GetEnvironmentVariable("CFLOOKUP_HangfireUser") ?? string.Empty;
8796
hangfirePassword = Environment.GetEnvironmentVariable("CFLOOKUP_HangfirePassword") ?? string.Empty;
97+
pgsqlConnString = Environment.GetEnvironmentVariable("CFLOOKUP_PGSQL") ?? string.Empty;
8898
}
8999

90100
var redis = ConnectionMultiplexer.Connect(redisServer);
91101

92102
builder.Services.AddSingleton(redis);
93103

94104
builder.Services.AddScoped(x => new SqlConnection(dbConnectionString));
105+
106+
builder.Services.AddScoped(x => new NpgsqlConnection(pgsqlConnString));
95107

96108
builder.Services.AddScoped<MSSQLDB>();
97109
#if !DEBUG
@@ -173,8 +185,13 @@ private static async Task Main(string[] args)
173185
RecurringJob.AddOrUpdate("cflookup:GetLatestUpdatedModPerGame", () => GetLatestUpdatedModPerGame.RunAsync(null), "*/5 * * * *");
174186
RecurringJob.AddOrUpdate("cflookup:SaveMinecraftModStats", () => SaveMinecraftModStats.RunAsync(null), Cron.Hourly());
175187
RecurringJob.AddOrUpdate("cflookup:CacheMCStatsOvertime", () => CacheMCOverTime.RunAsync(null), "*/30 * * * *");
188+
189+
if (!SharedMethods.CheckIfTaskIsScheduledOrInProgress("StoreCFApiProjects", "RunAsync"))
190+
{
191+
BackgroundJob.Schedule(() => StoreCFApiProjects.RunAsync(null), TimeSpan.FromSeconds(10));
192+
}
176193
#endif
177194

178-
app.Run();
195+
await app.RunAsync();
179196
}
180197
}

0 commit comments

Comments
 (0)