Skip to content

Commit 6654053

Browse files
itssimpleitssimple
committed
feat: Store all files as well
1 parent e48bcc6 commit 6654053

File tree

4 files changed

+351
-23
lines changed

4 files changed

+351
-23
lines changed

‎CFLookup/Jobs/StoreCFApiFiles.cs‎

Lines changed: 327 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,327 @@
1+
using CurseForge.APIClient;
2+
using CurseForge.APIClient.Models.Files;
3+
using CurseForge.APIClient.Models.Mods;
4+
using Hangfire;
5+
using Hangfire.Server;
6+
using Npgsql;
7+
using System.Text;
8+
using System.Text.Json;
9+
10+
namespace CFLookup.Jobs
11+
{
12+
public class StoreCFApiFiles
13+
{
14+
const int BUCKET_SIZE = 10_000;
15+
private const int EMPTY_BUCKETS = 300;
16+
private const int RETRY_BATCH = 3;
17+
18+
public async static Task RunAsync(PerformContext context)
19+
{
20+
using (var scope = Program.ServiceProvider.CreateScope())
21+
{
22+
try
23+
{
24+
var cfClient = scope.ServiceProvider.GetRequiredService<ApiClient>();
25+
cfClient.RequestDelay = TimeSpan.FromSeconds(0.05);
26+
cfClient.RequestTimeout = TimeSpan.FromMinutes(5);
27+
28+
var conn = scope.ServiceProvider.GetRequiredService<NpgsqlConnection>();
29+
await conn.OpenAsync();
30+
31+
var emptyBuckets = 0;
32+
33+
var buckets = SharedMethods.GetBucketRanges(1, int.MaxValue, BUCKET_SIZE);
34+
35+
foreach (var bucket in buckets)
36+
{
37+
var _bucket = Enumerable.Range(bucket.start, bucket.items);
38+
39+
var modList = await cfClient.GetFilesAsync(new GetModFilesRequestBody
40+
{
41+
FileIds = _bucket.ToList()
42+
});
43+
44+
if (modList.Error != null && modList.Error.ErrorCode != 404)
45+
{
46+
// No-op for now, maybe Discord logs later
47+
await SendDiscordErrorNotification(scope, $"The CF API threw an error at me: **{modList.Error.ErrorCode}**: {modList.Error.ErrorMessage}");
48+
}
49+
50+
if (modList.Data.Count == 0)
51+
{
52+
emptyBuckets++;
53+
if (emptyBuckets >= EMPTY_BUCKETS)
54+
{
55+
break;
56+
}
57+
}
58+
else
59+
{
60+
emptyBuckets = 0;
61+
}
62+
63+
await using var tx = await conn.BeginTransactionAsync();
64+
await using var batch = new NpgsqlBatch(conn);
65+
batch.Transaction = tx;
66+
batch.Timeout = 600;
67+
68+
foreach (var mod in modList.Data)
69+
{
70+
var cmd = new NpgsqlBatchCommand("""
71+
72+
INSERT INTO file_data (
73+
fileid,
74+
gameid,
75+
projectid,
76+
isavailable,
77+
displayname,
78+
filename,
79+
releasetype,
80+
filestatus,
81+
hashes,
82+
filedate,
83+
filelength,
84+
filesizeondisk,
85+
downloadcount,
86+
downloadurl,
87+
gameversions,
88+
sortablegameversions,
89+
dependencies,
90+
exposeasalternative,
91+
parentprojectfileid,
92+
alternatefileid,
93+
isserverpack,
94+
serverpackfileid,
95+
isearlyaccesscontent,
96+
earlyaccessenddate,
97+
filefingerprint,
98+
modules
99+
)
100+
VALUES (
101+
$1,
102+
$2,
103+
$3,
104+
$4,
105+
$5,
106+
$6,
107+
$7,
108+
$8,
109+
$9,
110+
$10,
111+
$11,
112+
$12,
113+
$13,
114+
$14,
115+
$15,
116+
$16,
117+
$17,
118+
$18,
119+
$19,
120+
$20,
121+
$21,
122+
$22,
123+
$23,
124+
$24,
125+
$25,
126+
$26
127+
)
128+
ON CONFLICT (fileid, gameid, projectid) DO UPDATE
129+
SET
130+
isavailable=EXCLUDED.isavailable,
131+
displayname=EXCLUDED.displayname,
132+
filename=EXCLUDED.filename,
133+
releasetype=EXCLUDED.releasetype,
134+
filestatus=EXCLUDED.filestatus,
135+
hashes=EXCLUDED.hashes,
136+
filedate=EXCLUDED.filedate,
137+
filelength=EXCLUDED.filelength,
138+
filesizeondisk=EXCLUDED.filesizeondisk,
139+
downloadcount=EXCLUDED.downloadcount,
140+
downloadurl=EXCLUDED.downloadurl,
141+
gameversions=EXCLUDED.gameversions,
142+
sortablegameversions=EXCLUDED.sortablegameversions,
143+
dependencies=EXCLUDED.dependencies,
144+
exposeasalternative=EXCLUDED.exposeasalternative,
145+
parentprojectfileid=EXCLUDED.parentprojectfileid,
146+
alternatefileid=EXCLUDED.alternatefileid,
147+
isserverpack=EXCLUDED.isserverpack,
148+
serverpackfileid=EXCLUDED.serverpackfileid,
149+
isearlyaccesscontent=EXCLUDED.isearlyaccesscontent,
150+
earlyaccessenddate=EXCLUDED.earlyaccessenddate,
151+
filefingerprint=EXCLUDED.filefingerprint,
152+
modules=EXCLUDED.modules,
153+
latestupdate=timezone('UTC'::text, now());
154+
""");
155+
156+
cmd.Parameters.AddWithValue(mod.Id);
157+
cmd.Parameters.AddWithValue(mod.GameId);
158+
cmd.Parameters.AddWithValue(mod.ModId);
159+
cmd.Parameters.AddWithValue(mod.IsAvailable);
160+
cmd.Parameters.AddWithValue(mod.DisplayName);
161+
cmd.Parameters.AddWithValue(mod.FileName);
162+
cmd.Parameters.AddWithValue((int)mod.ReleaseType);
163+
cmd.Parameters.AddWithValue((int)mod.FileStatus);
164+
cmd.Parameters.Add(new NpgsqlParameter()
165+
{
166+
NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Jsonb,
167+
Value = JsonSerializer.Serialize(mod.Hashes)
168+
});
169+
cmd.Parameters.AddWithValue(mod.FileDate);
170+
cmd.Parameters.AddWithValue(mod.FileLength);
171+
cmd.Parameters.Add(new NpgsqlParameter
172+
{
173+
NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Bigint,
174+
Value = (object?)mod.FileSizeOnDisk ?? DBNull.Value
175+
});
176+
cmd.Parameters.AddWithValue(mod.DownloadCount);
177+
cmd.Parameters.AddWithValue(mod.DownloadUrl ?? string.Empty);
178+
cmd.Parameters.Add(new NpgsqlParameter()
179+
{
180+
NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Jsonb,
181+
Value = JsonSerializer.Serialize(mod.GameVersions)
182+
});
183+
cmd.Parameters.Add(new NpgsqlParameter()
184+
{
185+
NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Jsonb,
186+
Value = JsonSerializer.Serialize(mod.SortableGameVersions)
187+
});
188+
cmd.Parameters.Add(new NpgsqlParameter()
189+
{
190+
NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Jsonb,
191+
Value = JsonSerializer.Serialize(mod.Dependencies)
192+
});
193+
194+
cmd.Parameters.Add(new NpgsqlParameter
195+
{
196+
NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Boolean,
197+
Value = (object?)mod.ExposeAsAlternative ?? DBNull.Value
198+
});
199+
cmd.Parameters.Add(new NpgsqlParameter
200+
{
201+
NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Integer,
202+
Value = (object?)mod.ParentProjectFileId ?? DBNull.Value
203+
});
204+
cmd.Parameters.Add(new NpgsqlParameter
205+
{
206+
NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Integer,
207+
Value = (object?)mod.AlternateFileId ?? DBNull.Value
208+
});
209+
cmd.Parameters.Add(new NpgsqlParameter
210+
{
211+
NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Boolean,
212+
Value = (object?)mod.IsServerPack ?? DBNull.Value
213+
});
214+
cmd.Parameters.Add(new NpgsqlParameter
215+
{
216+
NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Integer,
217+
Value = (object?)mod.ServerPackFileId ?? DBNull.Value
218+
});
219+
cmd.Parameters.Add(new NpgsqlParameter
220+
{
221+
NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Boolean,
222+
Value = (object?)mod.IsEarlyAccessContent ?? DBNull.Value
223+
});
224+
cmd.Parameters.Add(new NpgsqlParameter
225+
{
226+
NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.TimestampTz,
227+
Value = (object?)mod.EarlyAccessEndDate ?? DBNull.Value
228+
});
229+
cmd.Parameters.AddWithValue(mod.FileFingerprint);
230+
cmd.Parameters.Add(new NpgsqlParameter()
231+
{
232+
NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Jsonb,
233+
Value = JsonSerializer.Serialize(mod.Modules)
234+
});
235+
236+
batch.BatchCommands.Add(cmd);
237+
238+
if (batch.BatchCommands.Count >= 1000)
239+
{
240+
if (!await ExecuteBatchWithRetries(batch))
241+
{
242+
// No-op for now, maybe Discord logs later
243+
}
244+
}
245+
}
246+
247+
if (batch.BatchCommands.Count > 0)
248+
{
249+
if (!await ExecuteBatchWithRetries(batch))
250+
{
251+
// No-op for now, maybe Discord logs later
252+
}
253+
}
254+
255+
await tx.CommitAsync();
256+
}
257+
258+
}
259+
catch (Exception ex)
260+
{
261+
await SendDiscordErrorNotification(scope, $"Exception: {ex}");
262+
}
263+
finally
264+
{
265+
BackgroundJob.Schedule(() => StoreCFApiFiles.RunAsync(null), TimeSpan.FromMinutes(30));
266+
}
267+
}
268+
}
269+
270+
private async static Task SendDiscordErrorNotification(IServiceScope scope, string webhookMessage)
271+
{
272+
try
273+
{
274+
var httpClient = scope.ServiceProvider.GetRequiredService<IHttpClientFactory>().CreateClient();
275+
var discordWebhook =
276+
Environment.GetEnvironmentVariable("DISCORD_WEBHOOK_PROJECT", EnvironmentVariableTarget.Machine) ??
277+
Environment.GetEnvironmentVariable("DISCORD_WEBHOOK_PROJECT", EnvironmentVariableTarget.User) ??
278+
Environment.GetEnvironmentVariable("DISCORD_WEBHOOK_PROJECT", EnvironmentVariableTarget.Process) ??
279+
string.Empty;
280+
281+
if (!string.IsNullOrWhiteSpace(discordWebhook))
282+
{
283+
var message =
284+
@$"An error occurred while trying to store all project files from CurseForge, the command will run again in 30 minutes.
285+
{webhookMessage}";
286+
var payload = new
287+
{
288+
content = message,
289+
flags = 4
290+
};
291+
292+
var json = JsonSerializer.Serialize(payload);
293+
var content = new StringContent(json, Encoding.UTF8, "application/json");
294+
await httpClient.PostAsync(discordWebhook, content);
295+
}
296+
}
297+
catch
298+
{
299+
// No-op
300+
}
301+
}
302+
303+
private async static Task<bool> ExecuteBatchWithRetries(NpgsqlBatch batch, int retries = RETRY_BATCH)
304+
{
305+
for (var attempt = 1; attempt <= retries; attempt++)
306+
{
307+
try
308+
{
309+
await batch.ExecuteNonQueryAsync();
310+
batch.BatchCommands.Clear();
311+
return true;
312+
}
313+
catch (Exception)
314+
{
315+
if (attempt == retries)
316+
{
317+
return false;
318+
}
319+
320+
await Task.Delay(1000 * attempt);
321+
}
322+
}
323+
324+
return false;
325+
}
326+
}
327+
}

‎CFLookup/Jobs/StoreCFApiProjects.cs‎

Lines changed: 1 addition & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ public async static Task RunAsync(PerformContext context)
3030

3131
var emptyBuckets = 0;
3232

33-
var buckets = GetBucketRanges(1, int.MaxValue);
33+
var buckets = SharedMethods.GetBucketRanges(1, int.MaxValue, BUCKET_SIZE);
3434

3535
foreach (var bucket in buckets)
3636
{
@@ -313,27 +313,5 @@ private async static Task<bool> ExecuteBatchWithRetries(NpgsqlBatch batch, int r
313313

314314
return false;
315315
}
316-
317-
private static List<(int start, int end, int items)> GetBucketRanges(int start, int max)
318-
{
319-
var buckets = new List<(int start, int end, int items)>();
320-
321-
while (start <= max)
322-
{
323-
var bucketSize = Math.Min(BUCKET_SIZE, max - start + 1);
324-
var bucket = Enumerable.Range(start, bucketSize);
325-
326-
buckets.Add((bucket.First(), bucket.Last(), bucket.Count()));
327-
start += bucketSize;
328-
329-
if (start < 0)
330-
{
331-
// We went around, stop it
332-
break;
333-
}
334-
}
335-
336-
return buckets;
337-
}
338316
}
339317
}

‎CFLookup/Program.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,6 +190,7 @@ private static async Task Main(string[] args)
190190
if (!SharedMethods.CheckIfTaskIsScheduledOrInProgress("StoreCFApiProjects", "RunAsync"))
191191
{
192192
BackgroundJob.Schedule(() => StoreCFApiProjects.RunAsync(null), TimeSpan.FromSeconds(10));
193+
BackgroundJob.Schedule(() => StoreCFApiFiles.RunAsync(null), TimeSpan.FromSeconds(10));
193194
}
194195
#endif
195196

0 commit comments

Comments
 (0)