make object directory a config settings

LeftofZen · LeftofZen · commit 68cb126ae467 · 2024-08-30T00:34:25.000+10:00
diff --git a/AvaGui/Models/ObjectEditorModel.cs b/AvaGui/Models/ObjectEditorModel.cs
@@ -12,7 +12,6 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
-using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Net.Http;
@@ -255,20 +254,6 @@ public bool TryLoadObject(FileSystemItem filesystemItem, out UiLocoFile? uiLocoF
 			return true;
 		}
 
-		async Task CreateIndex(string[] allFiles, IProgress<float> progress)
-		{
-			Logger.Info($"Creating index on {allFiles.Length} files");
-
-			var sw = new Stopwatch();
-			sw.Start();
-
-			var fileCount = allFiles.Length;
-			ObjectIndex = await ObjectIndex.CreateIndexAsync(allFiles, progress);
-
-			sw.Stop();
-			Logger.Info($"Indexed {fileCount} in {sw.Elapsed}");
-		}
-
 		public bool LoadDataDirectory(string directory)
 		{
 			if (!Directory.Exists(directory))
@@ -373,26 +358,26 @@ async Task LoadObjDirectoryAsyncCore(string directory, IProgress<float> progress
 				if (exception || ObjectIndex?.Objects == null || ObjectIndex.Objects.Any(x => string.IsNullOrEmpty(x.Filename) || (x is ObjectIndexEntry xx && string.IsNullOrEmpty(xx.ObjectName))) != false)
 				{
 					Logger.Warning("Index file format has changed or otherwise appears to be malformed - recreating now.");
-					await RecreateIndex(progress, allFiles);
+					await RecreateIndex(directory, allFiles, progress);
 					return;
 				}
 
 				var objectIndexFilenames = ObjectIndex.Objects.Select(x => x.Filename).Concat(ObjectIndex.ObjectsFailed.Select(x => x.Filename));
 				if (objectIndexFilenames.Except(allFiles).Any() || allFiles.Except(objectIndexFilenames).Any())
 				{
 					Logger.Warning("Index file appears to be outdated - recreating now.");
-					await RecreateIndex(progress, allFiles);
+					await RecreateIndex(directory, allFiles, progress);
 				}
 			}
 			else
 			{
-				await RecreateIndex(progress, allFiles);
+				await RecreateIndex(directory, allFiles, progress);
 			}
 
-			async Task RecreateIndex(IProgress<float> progress, string[] allFiles)
+			async Task RecreateIndex(string rootObjectDirectory, string[] allFiles, IProgress<float> progress)
 			{
 				Logger.Info("Recreating index file");
-				await CreateIndex(allFiles, progress); // do we need the array?
+				ObjectIndex = await ObjectIndex.CreateIndexAsync(rootObjectDirectory, allFiles, progress);
 				ObjectIndex?.SaveIndex(IndexFilename);
 			}
 		}
diff --git a/Common/EditorSettings.cs b/Common/EditorSettings.cs
@@ -60,11 +60,10 @@ public HashSet<string> SCV5Directories
 		HashSet<string> scv5Directories;
 
 		public bool UseHttps { get; set; }
-		public string ServerAddressHttp { get; set; } = "http://www.leftofzen.dev/";
-		public string ServerAddressHttps { get; set; } = "https://www.leftofzen.dev/";
+		public string ServerAddressHttp { get; set; } = "http://openloco.leftofzen.dev/";
+		public string ServerAddressHttps { get; set; } = "https://openloco.leftofzen.dev/";
 
 		public string PaletteFile { get; set; } = "palette.png";
-		public string MetadataFileName { get; set; } = "objectMetadata.json";
 		public string IndexFileName { get; set; } = "objectIndex.json";
 		public string G1DatFileName { get; set; } = "g1.DAT";
 		public string DownloadFolder { get; set; } = string.Empty;
diff --git a/Dat/FileParsing/SawyerStreamUtils.cs b/Dat/FileParsing/SawyerStreamUtils.cs
@@ -24,9 +24,11 @@ static uint32_t ComputeChecksum(ReadOnlySpan<byte> data, uint32_t seed)
 			return checksum;
 		}
 
+		// returns paths relative to the input directory
 		public static IEnumerable<string> GetDatFilesInDirectory(string directory)
 			=> Directory
 				.GetFiles(directory, "*", SearchOption.AllDirectories)
-				.Where(x => Path.GetExtension(x).Equals(".dat", StringComparison.OrdinalIgnoreCase));
+				.Where(x => Path.GetExtension(x).Equals(".dat", StringComparison.OrdinalIgnoreCase))
+				.Select(x => Path.GetRelativePath(directory, x));
 	}
 }
diff --git a/Dat/ObjectIndex.cs b/Dat/ObjectIndex.cs
@@ -27,15 +27,15 @@ public void AddObject(ObjectIndexEntryBase entryBase)
 			}
 		}
 
-		public static Task<ObjectIndex> CreateIndexAsync(string[] files, IProgress<float>? progress)
+		public static Task<ObjectIndex> CreateIndexAsync(string rootObjectDirectory, string[] files, IProgress<float>? progress)
 		{
 			ConcurrentQueue<(string Filename, byte[] Data)> pendingFiles = [];
 			ConcurrentQueue<ObjectIndexEntryBase> pendingIndices = [];
 
 			var producerTask = Task.Run(async () =>
 			{
 				var options = new ParallelOptions() { MaxDegreeOfParallelism = 32 };
-				await Parallel.ForEachAsync(files, options, async (f, ct) => pendingFiles.Enqueue((f, await File.ReadAllBytesAsync(f, ct))));
+				await Parallel.ForEachAsync(files, options, async (f, ct) => pendingFiles.Enqueue((f, await File.ReadAllBytesAsync(Path.Combine(rootObjectDirectory, f), ct))));
 			});
 
 			var consumerTask = Task.Run(async () =>
@@ -80,7 +80,7 @@ public static ObjectIndex LoadOrCreateIndex(string directory)
 			else
 			{
 				var fileArr = SawyerStreamUtils.GetDatFilesInDirectory(directory).ToArray();
-				index = CreateIndexAsync(fileArr, null).Result;
+				index = CreateIndexAsync(directory, fileArr, null).Result;
 				index.SaveIndex(indexPath);
 			}
 
diff --git a/ObjectService/Program.cs b/ObjectService/Program.cs
@@ -48,6 +48,9 @@
 		};
 	}));
 
+builder.Services.AddSingleton<Server>();
+var serviceSettings = builder.Services.Configure<ObjectServiceSettings>(builder.Configuration.GetSection("ObjectService"));
+
 builder.Logging.ClearProviders();
 builder.Logging.AddConsole();
 
@@ -61,20 +64,23 @@
 	_ = app.UseSwaggerUI();
 }
 
+var objRoot = builder.Configuration["ObjectService:ObjectRootFolder"];
+var server = new Server(new ObjectServiceSettings() { ObjectRootFolder = objRoot! });
+
 // GET
 _ = app.MapGet(Routes.ListObjects, Server.ListObjects)
 	.RequireRateLimiting(tokenPolicy);
 
-_ = app.MapGet(Routes.GetDat, Server.GetDat)
+_ = app.MapGet(Routes.GetDat, server.GetDat)
 	.RequireRateLimiting(tokenPolicy);
 
-_ = app.MapGet(Routes.GetObject, Server.GetObject)
+_ = app.MapGet(Routes.GetObject, server.GetObject)
 	.RequireRateLimiting(tokenPolicy);
 
-_ = app.MapGet(Routes.GetDatFile, Server.GetDatFile)
+_ = app.MapGet(Routes.GetDatFile, server.GetDatFile)
 	.RequireRateLimiting(tokenPolicy);
 
-_ = app.MapGet(Routes.GetObjectFile, Server.GetObjectFile)
+_ = app.MapGet(Routes.GetObjectFile, server.GetObjectFile)
 	.RequireRateLimiting(tokenPolicy);
 
 // PATCH
@@ -85,7 +91,7 @@
 	.RequireRateLimiting(tokenPolicy);
 
 // POST
-_ = app.MapPost(Routes.UploadDat, Server.UploadDat)
+_ = app.MapPost(Routes.UploadDat, server.UploadDat)
 	.RequireRateLimiting(tokenPolicy);
 
 _ = app.MapPost(Routes.UploadObject, /*Server.UploadDat*/ () => Results.Problem(statusCode: StatusCodes.Status501NotImplemented))
diff --git a/ObjectService/Server.cs b/ObjectService/Server.cs
@@ -1,18 +1,28 @@
-using Microsoft.AspNetCore.Http;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Options;
 using OpenLoco.Dat.Data;
 using OpenLoco.Dat.FileParsing;
 using OpenLoco.Dat.Objects;
+using OpenLoco.Definitions;
 using OpenLoco.Definitions.Database;
 using OpenLoco.Definitions.DTO;
+using OpenLoco.Definitions.Web;
 
-namespace OpenLoco.Definitions.Web
+namespace OpenLoco.ObjectService
 {
 	// this must be done because eager-loading related many-to-many data in entity framework is recursive and cannot be turned off...
 	internal record ExpandedTblLocoObject(TblLocoObject Object, ICollection<TblAuthor> Authors, ICollection<TblTag> Tags, ICollection<TblModpack> Modpacks);
 
-	public static class Server
+	public class ObjectServiceSettings
 	{
+		public required string ObjectRootFolder { get; set; }
+	}
+
+	public class Server(ObjectServiceSettings settings)
+	{
+		public Server(IOptions<ObjectServiceSettings> options) : this(options.Value)
+		{ }
+
 		// eg: https://localhost:7230/objects/list
 		public static async Task<IResult> ListObjects(LocoDb db)
 			=> Results.Ok(
@@ -27,7 +37,7 @@ await db.Objects
 					x.VehicleType)).ToListAsync());
 
 		// eg: https://localhost:7230/objects/getdat?objectName=114&checksum=123$returnObjBytes=false
-		public static async Task<IResult> GetDat(string objectName, uint checksum, bool? returnObjBytes, LocoDb db)
+		public async Task<IResult> GetDat(string objectName, uint checksum, bool? returnObjBytes, LocoDb db)
 		{
 			var eObj = await db.Objects
 				.Where(x => x.OriginalName == objectName && x.OriginalChecksum == checksum)
@@ -37,11 +47,11 @@ public static async Task<IResult> GetDat(string objectName, uint checksum, bool?
 
 			return eObj == null || eObj.Object == null
 				? Results.NotFound()
-				: Results.Ok(await PrepareLocoObject(eObj, returnObjBytes ?? false));
+				: Results.Ok(await PrepareLocoObject(eObj, settings.ObjectRootFolder, returnObjBytes ?? false));
 		}
 
 		// eg: https://localhost:7230/objects/getobject?uniqueObjectId=246263256&returnObjBytes=false
-		public static async Task<IResult> GetObject(int uniqueObjectId, bool? returnObjBytes, LocoDb db)
+		public async Task<IResult> GetObject(int uniqueObjectId, bool? returnObjBytes, LocoDb db)
 		{
 			Console.WriteLine($"Object [{uniqueObjectId}] requested");
 			var eObj = await db.Objects
@@ -52,11 +62,11 @@ public static async Task<IResult> GetObject(int uniqueObjectId, bool? returnObjB
 
 			return eObj == null || eObj.Object == null
 				? Results.NotFound()
-				: Results.Ok(await PrepareLocoObject(eObj, returnObjBytes ?? false));
+				: Results.Ok(await PrepareLocoObject(eObj, settings.ObjectRootFolder, returnObjBytes ?? false));
 		}
 
 		// eg: https://localhost:7230/objects/originaldatfile?objectName=114&checksum=123
-		public static async Task<IResult> GetDatFile(string objectName, uint checksum, LocoDb db)
+		public async Task<IResult> GetDatFile(string objectName, uint checksum, LocoDb db)
 		{
 			var obj = await db.Objects
 				.Where(x => x.OriginalName == objectName && x.OriginalChecksum == checksum)
@@ -70,23 +80,36 @@ public static async Task<IResult> GetDatFile(string objectName, uint checksum, L
 		}
 
 		// eg: https://localhost:7230/objects/getobjectfile?objectName=114&checksum=123
-		public static async Task<IResult> GetObjectFile(int uniqueObjectId, LocoDb db)
+		public async Task<IResult> GetObjectFile(int uniqueObjectId, LocoDb db)
 		{
 			var obj = await db.Objects
 				.Where(x => x.Id == uniqueObjectId)
 				.SingleOrDefaultAsync();
 
 			const string contentType = "application/octet-stream";
 
-			return obj?.IsVanilla == false && File.Exists(obj.PathOnDisk)
-				? Results.File(obj.PathOnDisk, contentType, Path.GetFileName(obj.PathOnDisk))
+			if (obj == null)
+			{
+				return Results.NotFound();
+			}
+
+			if (obj.IsVanilla)
+			{
+				return Results.BadRequest("Nice try genius. Downloading vanilla objects is not allowed.");
+			}
+
+			var pathOnDisk = Path.Combine(settings.ObjectRootFolder, obj.PathOnDisk);
+			return File.Exists(pathOnDisk)
+				? Results.File(pathOnDisk, contentType, Path.GetFileName(pathOnDisk))
 				: Results.NotFound();
 		}
 
-		internal static async Task<DtoLocoObject> PrepareLocoObject(ExpandedTblLocoObject eObj, bool returnObjBytes)
+		internal static async Task<DtoLocoObject> PrepareLocoObject(ExpandedTblLocoObject eObj, string objectRootFolder, bool returnObjBytes)
 		{
 			var obj = eObj!.Object;
-			var bytes = returnObjBytes && !obj.IsVanilla && File.Exists(obj.PathOnDisk) ? Convert.ToBase64String(await File.ReadAllBytesAsync(obj.PathOnDisk)) : null;
+
+			var pathOnDisk = Path.Combine(objectRootFolder, obj.PathOnDisk);
+			var bytes = returnObjBytes && !obj.IsVanilla && File.Exists(pathOnDisk) ? Convert.ToBase64String(await File.ReadAllBytesAsync(pathOnDisk)) : null;
 
 			return new DtoLocoObject(
 				obj.Id,
@@ -109,7 +132,7 @@ internal static async Task<DtoLocoObject> PrepareLocoObject(ExpandedTblLocoObjec
 		}
 
 		// eg: <todo>
-		public static async Task<IResult> UploadDat(DtoUploadDat request, LocoDb db)
+		public async Task<IResult> UploadDat(DtoUploadDat request, LocoDb db)
 		{
 			if (string.IsNullOrEmpty(request.DatBytesAsBase64))
 			{
@@ -158,15 +181,17 @@ public static async Task<IResult> UploadDat(DtoUploadDat request, LocoDb db)
 				return Results.Accepted($"Object already exists in the database. OriginalName={s5Header.Name} OriginalChecksum={s5Header.Checksum} UploadDate={existingObject!.UploadDate}");
 			}
 
-			const string UploadFolder = "Q:\\Games\\Locomotion\\Server\\CustomObjects\\Uploaded";
+			const string SettingsPath = "Q:\\Games\\Locomotion\\Server\\CustomObjects";
+			const string UploadFolder = "UploadedObjects";
 			var uuid = Guid.NewGuid();
-			var saveFileName = Path.Combine(UploadFolder, $"{uuid}.dat");
+			var saveFileName = Path.Combine(SettingsPath, UploadFolder, $"{uuid}.dat");
+			File.WriteAllBytes(saveFileName, datFileBytes);
+
 			Console.WriteLine($"File accepted OriginalName={s5Header.Name} OriginalChecksum={s5Header.Checksum} PathOnDisk={saveFileName}");
 
 			var locoTbl = new TblLocoObject()
 			{
 				Name = $"{s5Header.Name}_{s5Header.Checksum}", // same as DB seeder name
-				PathOnDisk = saveFileName,
 				OriginalName = s5Header.Name,
 				OriginalChecksum = s5Header.Checksum,
 				IsVanilla = false, // not possible to upload vanilla objects
diff --git a/ObjectService/appsettings.json b/ObjectService/appsettings.json
@@ -20,5 +20,8 @@
   "ConnectionStrings": {
     "SQLiteConnection": "Data Source=Q:\\Games\\Locomotion\\Server\\loco-dev.db"
 
+  },
+  "ObjectService": {
+    "ObjectRootFolder": "Q:\\Games\\Locomotion\\Server\\Objects"
   }
 }

Original file line number	Diff line number	Diff line change
`@@ -12,7 +12,6 @@`
`12`	`12`	`using System;`
`13`	`13`	`using System.Collections.Generic;`
`14`	`14`	`using System.Collections.ObjectModel;`
`15`		`-using System.Diagnostics;`
`16`	`15`	`using System.IO;`
`17`	`16`	`using System.Linq;`
`18`	`17`	`using System.Net.Http;`
`@@ -255,20 +254,6 @@ public bool TryLoadObject(FileSystemItem filesystemItem, out UiLocoFile? uiLocoF`
`255`	`254`	`return true;`
`256`	`255`	`}`
`257`	`256`
`258`		`- async Task CreateIndex(string[] allFiles, IProgress<float> progress)`
`259`		`- {`
`260`		`- Logger.Info($"Creating index on {allFiles.Length} files");`
`261`		`-`
`262`		`- var sw = new Stopwatch();`
`263`		`- sw.Start();`
`264`		`-`
`265`		`- var fileCount = allFiles.Length;`
`266`		`- ObjectIndex = await ObjectIndex.CreateIndexAsync(allFiles, progress);`
`267`		`-`
`268`		`- sw.Stop();`
`269`		`- Logger.Info($"Indexed {fileCount} in {sw.Elapsed}");`
`270`		`- }`
`271`		`-`
`272`	`257`	`public bool LoadDataDirectory(string directory)`
`273`	`258`	`{`
`274`	`259`	`if (!Directory.Exists(directory))`
`@@ -373,26 +358,26 @@ async Task LoadObjDirectoryAsyncCore(string directory, IProgress<float> progress`
`373`	`358`	`if (exception \|\| ObjectIndex?.Objects == null \|\| ObjectIndex.Objects.Any(x => string.IsNullOrEmpty(x.Filename) \|\| (x is ObjectIndexEntry xx && string.IsNullOrEmpty(xx.ObjectName))) != false)`
`374`	`359`	`{`
`375`	`360`	`Logger.Warning("Index file format has changed or otherwise appears to be malformed - recreating now.");`
`376`		`- await RecreateIndex(progress, allFiles);`
	`361`	`+ await RecreateIndex(directory, allFiles, progress);`
`377`	`362`	`return;`
`378`	`363`	`}`
`379`	`364`
`380`	`365`	`var objectIndexFilenames = ObjectIndex.Objects.Select(x => x.Filename).Concat(ObjectIndex.ObjectsFailed.Select(x => x.Filename));`
`381`	`366`	`if (objectIndexFilenames.Except(allFiles).Any() \|\| allFiles.Except(objectIndexFilenames).Any())`
`382`	`367`	`{`
`383`	`368`	`Logger.Warning("Index file appears to be outdated - recreating now.");`
`384`		`- await RecreateIndex(progress, allFiles);`
	`369`	`+ await RecreateIndex(directory, allFiles, progress);`
`385`	`370`	`}`
`386`	`371`	`}`
`387`	`372`	`else`
`388`	`373`	`{`
`389`		`- await RecreateIndex(progress, allFiles);`
	`374`	`+ await RecreateIndex(directory, allFiles, progress);`
`390`	`375`	`}`
`391`	`376`
`392`		`- async Task RecreateIndex(IProgress<float> progress, string[] allFiles)`
	`377`	`+ async Task RecreateIndex(string rootObjectDirectory, string[] allFiles, IProgress<float> progress)`
`393`	`378`	`{`
`394`	`379`	`Logger.Info("Recreating index file");`
`395`		`- await CreateIndex(allFiles, progress); // do we need the array?`
	`380`	`+ ObjectIndex = await ObjectIndex.CreateIndexAsync(rootObjectDirectory, allFiles, progress);`
`396`	`381`	`ObjectIndex?.SaveIndex(IndexFilename);`
`397`	`382`	`}`
`398`	`383`	`}`
Original file line number	Diff line number	Diff line change
`@@ -24,9 +24,11 @@ static uint32_t ComputeChecksum(ReadOnlySpan<byte> data, uint32_t seed)`
`24`	`24`	`return checksum;`
`25`	`25`	`}`
`26`	`26`
	`27`	`+ // returns paths relative to the input directory`
`27`	`28`	`public static IEnumerable<string> GetDatFilesInDirectory(string directory)`
`28`	`29`	`=> Directory`
`29`	`30`	`.GetFiles(directory, "*", SearchOption.AllDirectories)`
`30`		`- .Where(x => Path.GetExtension(x).Equals(".dat", StringComparison.OrdinalIgnoreCase));`
	`31`	`+ .Where(x => Path.GetExtension(x).Equals(".dat", StringComparison.OrdinalIgnoreCase))`
	`32`	`+ .Select(x => Path.GetRelativePath(directory, x));`
`31`	`33`	`}`
`32`	`34`	`}`
Original file line number	Diff line number	Diff line change
`@@ -27,15 +27,15 @@ public void AddObject(ObjectIndexEntryBase entryBase)`
`27`	`27`	`}`
`28`	`28`	`}`
`29`	`29`
`30`		`- public static Task<ObjectIndex> CreateIndexAsync(string[] files, IProgress<float>? progress)`
	`30`	`+ public static Task<ObjectIndex> CreateIndexAsync(string rootObjectDirectory, string[] files, IProgress<float>? progress)`
`31`	`31`	`{`
`32`	`32`	`ConcurrentQueue<(string Filename, byte[] Data)> pendingFiles = [];`
`33`	`33`	`ConcurrentQueue<ObjectIndexEntryBase> pendingIndices = [];`
`34`	`34`
`35`	`35`	`var producerTask = Task.Run(async () =>`
`36`	`36`	`{`
`37`	`37`	`var options = new ParallelOptions() { MaxDegreeOfParallelism = 32 };`
`38`		`- await Parallel.ForEachAsync(files, options, async (f, ct) => pendingFiles.Enqueue((f, await File.ReadAllBytesAsync(f, ct))));`
	`38`	`+ await Parallel.ForEachAsync(files, options, async (f, ct) => pendingFiles.Enqueue((f, await File.ReadAllBytesAsync(Path.Combine(rootObjectDirectory, f), ct))));`
`39`	`39`	`});`
`40`	`40`
`41`	`41`	`var consumerTask = Task.Run(async () =>`
`@@ -80,7 +80,7 @@ public static ObjectIndex LoadOrCreateIndex(string directory)`
`80`	`80`	`else`
`81`	`81`	`{`
`82`	`82`	`var fileArr = SawyerStreamUtils.GetDatFilesInDirectory(directory).ToArray();`
`83`		`- index = CreateIndexAsync(fileArr, null).Result;`
	`83`	`+ index = CreateIndexAsync(directory, fileArr, null).Result;`
`84`	`84`	`index.SaveIndex(indexPath);`
`85`	`85`	`}`
`86`	`86`
Original file line number	Diff line number	Diff line change
`@@ -20,5 +20,8 @@`
`20`	`20`	`"ConnectionStrings": {`
`21`	`21`	`"SQLiteConnection": "Data Source=Q:\\Games\\Locomotion\\Server\\loco-dev.db"`
`22`	`22`
	`23`	`+ },`
	`24`	`+ "ObjectService": {`
	`25`	`+ "ObjectRootFolder": "Q:\\Games\\Locomotion\\Server\\Objects"`
`23`	`26`	`}`
`24`	`27`	`}`