Skip to content

Commit c9d3380

Browse files
LeftofZenLeftofZen
authored
Sanitise data and improve object service (#103)
* add project and start cleaning * use object index on server * large reworks of the object service * increase rate limits, add http error message log * reenable log to file * cleanup merge * add availability to metadata * add back migrations
1 parent 4c16188 commit c9d3380

32 files changed

+1306
-370
lines changed

‎AvaGui/Models/Metadata.cs‎ renamed to ‎AvaGui/Models/MetadataModel.cs‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,23 +8,28 @@ namespace AvaGui.Models
88
{
99
public class MetadataModel
1010
{
11-
public MetadataModel(string originalName, uint originalChecksum)
11+
public MetadataModel(string uniqueName, string datName, uint datChecksum)
1212
{
13-
DatName = originalName;
14-
DatChecksum = originalChecksum;
13+
UniqueName = uniqueName;
14+
DatName = datName;
15+
DatChecksum = datChecksum;
1516
}
1617

17-
public string DatName { get; }
18-
public uint DatChecksum { get; }
18+
public string UniqueName { get; init; }
19+
20+
public string DatName { get; init; }
21+
public uint DatChecksum { get; init; }
1922
public string? Description { get; set; }
2023

24+
[Browsable(false)]
2125
public ICollection<TblAuthor> Authors { get; set; }
2226

2327
public DateTimeOffset? CreationDate { get; set; }
2428

2529
public DateTimeOffset? LastEditDate { get; set; }
2630

27-
public DateTimeOffset? UploadDate { get; set; }
31+
public DateTimeOffset UploadDate { get; set; }
32+
2833
[Browsable(false)]
2934

3035
public ICollection<TblTag> Tags { get; set; }

‎AvaGui/Models/ObjectEditorModel.cs‎

Lines changed: 43 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
using Avalonia.Threading;
2-
using Dat;
32
using DynamicData;
43
using OpenLoco.Common;
54
using OpenLoco.Common.Logging;
@@ -34,7 +33,7 @@ public class ObjectEditorModel
3433

3534
public ObjectIndex? ObjectIndexOnline { get; set; }
3635

37-
public Dictionary<int, DtoLocoObject> OnlineCache { get; } = [];
36+
public Dictionary<int, DtoDatObjectWithMetadata> OnlineCache { get; } = [];
3837

3938
public PaletteMap PaletteMap { get; set; }
4039

@@ -199,17 +198,16 @@ public bool TryLoadObject(FileSystemItem filesystemItem, out UiLocoFile? uiLocoF
199198

200199
if (locoObj == null)
201200
{
202-
Logger.Error($"Unable to object {filesystemItem.Name} with unique id {uniqueObjectId} from online - received no data");
201+
Logger.Error($"Unable to download object {filesystemItem.Name} with unique id {uniqueObjectId} from online - received no data");
203202
return false;
204203
}
205-
else if (locoObj.IsVanilla)
204+
else if (string.IsNullOrEmpty(locoObj.DatBytes))
206205
{
207-
Logger.Info($"Unable to object {filesystemItem.Name} with unique id {uniqueObjectId} from online - requested object is a vanilla object and it is illegal to distribute copyright material");
208-
return false;
206+
Logger.Warning($"Unable to download object {filesystemItem.Name} with unique id {uniqueObjectId} from online - received no DAT object data. Will still show metadata");
209207
}
210-
else if (string.IsNullOrEmpty(locoObj.OriginalBytes))
208+
else if (locoObj.IsVanilla)
211209
{
212-
Logger.Warning($"Unable to load object {filesystemItem.Name} with unique id {uniqueObjectId} from online - received no DAT object data");
210+
Logger.Warning($"Unable to download object {filesystemItem.Name} with unique id {uniqueObjectId} from online - requested object is a vanilla object and it is illegal to distribute copyright material. Will still show metadata");
213211
}
214212

215213
Logger.Info($"Downloaded object {filesystemItem.Name} with unique id {uniqueObjectId} and added it to the local cache");
@@ -221,38 +219,38 @@ public bool TryLoadObject(FileSystemItem filesystemItem, out UiLocoFile? uiLocoF
221219
Logger.Debug($"Found object {filesystemItem.Name} with unique id {uniqueObjectId} in cache - reusing it");
222220
}
223221

224-
if (locoObj != null && locoObj.OriginalBytes?.Length > 0)
222+
if (locoObj != null)
225223
{
226-
var obj = SawyerStreamReader.LoadFullObjectFromStream(Convert.FromBase64String(locoObj.OriginalBytes), Logger, $"{filesystemItem.Filename}-{filesystemItem.Name}", true);
227-
if (obj == null)
228-
{
229-
Logger.Warning($"Unable to load {filesystemItem.Name} from the received DAT object data");
230-
}
231-
else
224+
if (locoObj.DatBytes?.Length > 0)
232225
{
233-
fileInfo = obj?.DatFileInfo;
234-
locoObject = obj?.LocoObject;
226+
var obj = SawyerStreamReader.LoadFullObjectFromStream(Convert.FromBase64String(locoObj.DatBytes), Logger, $"{filesystemItem.Filename}-{filesystemItem.Name}", true);
227+
fileInfo = obj.DatFileInfo;
228+
locoObject = obj.LocoObject;
229+
if (obj.LocoObject == null)
230+
{
231+
Logger.Warning($"Unable to load {filesystemItem.Name} from the received DAT object data");
232+
}
235233
}
236-
}
237234

238-
metadata = new MetadataModel(locoObj.DatName, locoObj.DatChecksum)
239-
{
240-
Description = locoObj.Description,
241-
Authors = locoObj.Authors,
242-
CreationDate = locoObj.CreationDate,
243-
LastEditDate = locoObj.LastEditDate,
244-
UploadDate = locoObj.UploadDate,
245-
Tags = locoObj.Tags,
246-
Modpacks = locoObj.Modpacks,
247-
Availability = locoObj.Availability,
248-
Licence = locoObj.Licence,
249-
};
250-
251-
if (locoObject != null)
252-
{
253-
foreach (var i in locoObject.G1Elements)
235+
metadata = new MetadataModel(locoObj.UniqueName, locoObj.DatName, locoObj.DatChecksum)
254236
{
255-
images.Add(PaletteMap.ConvertG1ToRgba32Bitmap(i));
237+
Description = locoObj.Description,
238+
Authors = locoObj.Authors,
239+
CreationDate = locoObj.CreationDate,
240+
LastEditDate = locoObj.LastEditDate,
241+
UploadDate = locoObj.UploadDate,
242+
Tags = locoObj.Tags,
243+
Modpacks = locoObj.Modpacks,
244+
Availability = locoObj.Availability,
245+
Licence = locoObj.Licence,
246+
};
247+
248+
if (locoObject != null)
249+
{
250+
foreach (var i in locoObject.G1Elements)
251+
{
252+
images.Add(PaletteMap.ConvertG1ToRgba32Bitmap(i));
253+
}
256254
}
257255
}
258256
}
@@ -376,7 +374,6 @@ async Task LoadObjDirectoryAsyncCore(string directory, IProgress<float> progress
376374

377375
Settings.ObjDataDirectory = directory;
378376
SaveSettings();
379-
var allFiles = SawyerStreamUtils.GetDatFilesInDirectory(directory).ToArray();
380377

381378
if (useExistingIndex && File.Exists(IndexFilename))
382379
{
@@ -395,26 +392,27 @@ async Task LoadObjDirectoryAsyncCore(string directory, IProgress<float> progress
395392
if (exception || ObjectIndex?.Objects == null || ObjectIndex.Objects.Any(x => string.IsNullOrEmpty(x.Filename) || (x is ObjectIndexEntry xx && string.IsNullOrEmpty(xx.DatName))))
396393
{
397394
Logger.Warning("Index file format has changed or otherwise appears to be malformed - recreating now.");
398-
await RecreateIndex(directory, allFiles, progress);
395+
await RecreateIndex(directory, progress);
399396
return;
400397
}
401398

402-
var objectIndexFilenames = ObjectIndex.Objects.Select(x => x.Filename).Concat(ObjectIndex.ObjectsFailed.Select(x => x.Filename));
399+
var objectIndexFilenames = ObjectIndex.Objects.Select(x => x.Filename);
400+
var allFiles = SawyerStreamUtils.GetDatFilesInDirectory(directory).ToArray();
403401
if (objectIndexFilenames.Except(allFiles).Any() || allFiles.Except(objectIndexFilenames).Any())
404402
{
405403
Logger.Warning("Index file appears to be outdated - recreating now.");
406-
await RecreateIndex(directory, allFiles, progress);
404+
await RecreateIndex(directory, progress);
407405
}
408406
}
409407
else
410408
{
411-
await RecreateIndex(directory, allFiles, progress);
409+
await RecreateIndex(directory, progress);
412410
}
413411

414-
async Task RecreateIndex(string rootObjectDirectory, string[] allFiles, IProgress<float> progress)
412+
async Task RecreateIndex(string directory, IProgress<float> progress)
415413
{
416414
Logger.Info("Recreating index file");
417-
ObjectIndex = await ObjectIndex.CreateIndexAsync(rootObjectDirectory, allFiles, progress);
415+
ObjectIndex = await ObjectIndex.CreateIndexAsync(directory, progress);
418416
ObjectIndex?.SaveIndex(IndexFilename);
419417
}
420418
}
@@ -429,8 +427,8 @@ public async Task CheckForDatFilesNotOnServer()
429427
Logger.Debug("Comparing local objects to object repository");
430428

431429
var localButNotOnline = ObjectIndex.Objects.ExceptBy(ObjectIndexOnline.Objects.Select(
432-
x => (x.DatName, x.Checksum)),
433-
x => (x.DatName, x.Checksum)).ToList();
430+
x => (x.DatName, x.DatChecksum)),
431+
x => (x.DatName, x.DatChecksum)).ToList();
434432

435433
if (localButNotOnline.Count != 0)
436434
{
@@ -455,6 +453,7 @@ public async Task UploadDatToServer(ObjectIndexEntry dat)
455453
var filename = Path.Combine(Settings.ObjDataDirectory, dat.Filename);
456454
var lastModifiedTime = File.GetLastWriteTimeUtc(filename); // this is the "Modified" time as shown in Windows
457455
await Client.UploadDatFileAsync(WebClient, dat.Filename, await File.ReadAllBytesAsync(filename), lastModifiedTime, Logger);
456+
await Task.Delay(100); // wait 100ms, ie don't DoS the server
458457
}
459458
}
460459
}

‎AvaGui/ViewModels/FolderTreeViewModel.cs‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
using AvaGui.Models;
22
using Avalonia.Controls;
3-
using Dat;
3+
using OpenLoco.Dat;
44
using OpenLoco.Dat.Data;
55
using OpenLoco.Definitions.Web;
66
using ReactiveUI;
@@ -174,9 +174,8 @@ async Task LoadOnlineDirectoryAsync(bool useExistingIndex)
174174
Model.ObjectIndexOnline = new ObjectIndex()
175175
{
176176
Objects = (await Client.GetObjectListAsync(Model.WebClient, Model.Logger))
177-
.Select(x => new ObjectIndexEntry(x.Id.ToString(), x.DatName, x.ObjectType, x.IsVanilla, x.DatChecksum, x.VehicleType))
178-
.ToList(),
179-
ObjectsFailed = []
177+
.Select(x => new ObjectIndexEntry(x.Id.ToString(), x.DatName, x.DatChecksum, x.ObjectType, x.IsVanilla, x.VehicleType))
178+
.ToList()
180179
};
181180
}
182181

@@ -190,7 +189,7 @@ async Task LoadOnlineDirectoryAsync(bool useExistingIndex)
190189
}
191190
}
192191

193-
static List<FileSystemItemBase> ConstructTreeView(IEnumerable<ObjectIndexEntryBase> index, string filenameFilter, ObjectDisplayMode displayMode, FileLocation fileLocation)
192+
static List<FileSystemItemBase> ConstructTreeView(IEnumerable<ObjectIndexEntry> index, string filenameFilter, ObjectDisplayMode displayMode, FileLocation fileLocation)
194193
{
195194
var result = new List<FileSystemItemBase>();
196195

‎Common/EditorSettings.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ public HashSet<string> SCV5Directories
6464
public string ServerAddressHttps { get; set; } = "https://openloco.leftofzen.dev/";
6565

6666
public string PaletteFile { get; set; } = "palette.png";
67-
public string IndexFileName { get; set; } = "objectIndex.json";
67+
public string IndexFileName { get; set; } = "objectIndex.json"; // this should be the same as ObjectIndex.IndexFileName
6868
public string G1DatFileName { get; set; } = "g1.DAT";
6969
public string DownloadFolder { get; set; } = string.Empty;
7070

‎Common/GlenDbSchema.cs‎

Lines changed: 0 additions & 62 deletions
This file was deleted.

‎Dat/Data/OriginalObjectFiles.cs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,26 @@
11
namespace OpenLoco.Dat.Data
22
{
3+
public enum FileSource { VanillaSteam, VanillaGoG, Custom };
4+
35
public static class OriginalObjectFiles
46
{
7+
public static FileSource GetFileSource(string name, uint checksum)
8+
{
9+
if (Names.TryGetValue(name, out var chksum))
10+
{
11+
if (checksum == chksum.SteamChecksum)
12+
{
13+
return FileSource.VanillaSteam;
14+
}
15+
else if (checksum == chksum.GoGChecksum)
16+
{
17+
return FileSource.VanillaGoG;
18+
}
19+
}
20+
21+
return FileSource.Custom;
22+
}
23+
524
public static readonly Dictionary<string, (uint SteamChecksum, uint GoGChecksum)> Names = new()
625
{
726
{ "114", (1733551639, 1873462701) },

0 commit comments

Comments
 (0)