Improve indexing performance (#60)

* fix a lot of analyzer warnings and suggestions

* performance optimisations and benchmarking

* make indexing async

* add a progress bar for indexing

Author: LeftofZen

AvaGui/Models/IndexObjectHeader.cs

@@ -54,7 +54,8 @@ public class ObjectEditorModel
 		public ObjectEditorModel()
 		{
 			Logger = new Logger();
-			LoggerObservableLogs = new ObservableCollection<LogLine>(((Logger)Logger).Logs);
+			LoggerObservableLogs = [];
+			Logger.LogAdded += (sender, laea) => LoggerObservableLogs.Add(laea.Log);
 
 			LoadSettings(SettingsFile, Logger);
 		}
@@ -76,15 +77,13 @@ public void LoadSettings(string settingsFile, ILogger? logger)
 
 			Settings = settings!;
 
-			if (!ValidateSettings(Settings, logger))
+			if (ValidateSettings(Settings, logger))
 			{
-				return;
-			}
-
-			if (File.Exists(Settings.GetObjDataFullPath(Settings.IndexFileName)))
-			{
-				logger?.Info($"Loading header index from \"{Settings.IndexFileName}\"");
-				LoadObjDirectory(Settings.ObjDataDirectory, new Progress<float>(), true);
+				if (File.Exists(Settings.GetObjDataFullPath(Settings.IndexFileName)))
+				{
+					Logger?.Info($"Loading header index from \"{Settings.IndexFileName}\"");
+					LoadObjDirectoryAsync(Settings.ObjDataDirectory, new Progress<float>(), true).Wait();
+				}
 			}
 		}
 
@@ -159,70 +158,29 @@ public bool TryLoadObject(string filename, out UiLocoFile? uiLocoFile)
 		}
 
 		// this method loads every single object entirely. it takes a long time to run
-		void CreateIndex(string[] allFiles, IProgress<float>? progress)
+		async Task CreateIndex(string[] allFiles, IProgress<float> progress)
 		{
 			Logger?.Info($"Creating index on {allFiles.Length} files");
 
-			ConcurrentDictionary<string, IndexObjectHeader> ccHeaderIndex = new(); // key is full path/filename
-
-			var count = 0;
-			ConcurrentDictionary<string, TimeSpan> timePerFile = new();
-
 			var sw = new Stopwatch();
 			sw.Start();
 
 			var fileCount = allFiles.Length;
-
-			_ = Parallel.ForEach(allFiles, new ParallelOptions() { MaxDegreeOfParallelism = 16 }, (filename)
-				=> count = LoadAndIndexFile(count, filename));
-
-			HeaderIndex = ccHeaderIndex.OrderBy(kvp => kvp.Key).ToDictionary(kvp => kvp.Key, kvp => kvp.Value);
+			var index = await SawyerStreamReader.FastIndexAsync(allFiles, progress);
+
+			HeaderIndex = index.ToDictionary(
+				x => x.filename,
+				x => new ObjectIndexModel(
+					x.s5.Name,
+					DatFileType.Object,
+					x.s5.ObjectType,
+					x.s5.SourceGame,
+					x.s5.Checksum,
+					x.VehicleType)
+				);
 
 			sw.Stop();
-			Logger?.Info("Finished creating index");
-			Logger?.Debug($"Time time={sw.Elapsed}");
-
-			if (timePerFile.IsEmpty)
-			{
-				_ = timePerFile.TryAdd("<no items>", TimeSpan.Zero);
-			}
-
-			var slowest = timePerFile.MaxBy(x => x.Value.Ticks);
-			Logger?.Debug($"Slowest file={slowest.Key} Time={slowest.Value}");
-
-			var average = timePerFile.Average(x => x.Value.TotalMilliseconds);
-			Logger?.Debug($"Average time={average}ms");
-
-			var median = timePerFile.OrderBy(x => x.Value).Skip(timePerFile.Count / 2).Take(1).Single();
-			Logger?.Debug($"Median time={median.Value}ms");
-
-			int LoadAndIndexFile(int count, string filename)
-			{
-				var startTime = sw.Elapsed;
-				var loadResult = TryLoadObject(filename, out var uiLocoFile);
-				var elapsed = sw.Elapsed - startTime;
-
-				if (loadResult && uiLocoFile != null)
-				{
-					_ = ccHeaderIndex.TryAdd(filename, new IndexObjectHeader(
-						uiLocoFile.DatFileInfo.S5Header.Name,
-						DatFileType.Object,
-						uiLocoFile.DatFileInfo.S5Header.ObjectType,
-						uiLocoFile.DatFileInfo.S5Header.SourceGame,
-						uiLocoFile.DatFileInfo.S5Header.Checksum,
-						uiLocoFile.LocoObject.Object is VehicleObject veh ? veh.Type : null));
-
-					_ = timePerFile.TryAdd(uiLocoFile.DatFileInfo.S5Header.Name, elapsed);
-				}
-				else
-				{
-					Logger?.Error($"Failed to load \"{filename}\"");
-				}
-
-				_ = Interlocked.Increment(ref count);
-				progress?.Report((float)count / fileCount);
-				return count;
-			}
+			Logger?.Info($"Indexed {fileCount} in {sw.Elapsed}");
 		}
 
 		public void SaveFile(string path, UiLocoFile obj)
@@ -275,7 +233,7 @@ void LoadKnownData(HashSet<string> allFilesInDir, HashSet<string> knownFilenames
 
 			//LoadPalette(); // update palette from g1
 
-			SaveSettings();
+			//await SaveSettings();
 
 			return true;
 		}
@@ -286,19 +244,14 @@ void LoadKnownData(HashSet<string> allFilesInDir, HashSet<string> knownFilenames
 		//	var allFiles = Directory.GetFiles(directory, "*.dat|*.sv5|*.sc5", SearchOption.AllDirectories);
 		//}
 
-		public async Task LoadObjDirectoryAsync(string directory, IProgress<float>? progress, bool useExistingIndex)
-		{
-			await Task.Run(() => LoadObjDirectory(directory, progress, useExistingIndex));
-			await Task.Run(SaveSettings);
-		}
-
-		public void LoadObjDirectory(string directory)
-			=> LoadObjDirectory(directory, null, true);
+		//public void LoadObjDirectory(string directory)
+		//	=> LoadObjDirectory(directory, new Progress<float>(), true);
 
-		public void LoadObjDirectory(string directory, IProgress<float>? progress, bool useExistingIndex)
+		public async Task LoadObjDirectoryAsync(string directory, IProgress<float> progress, bool useExistingIndex)
 		{
-			if (!Directory.Exists(directory))
+			if (string.IsNullOrEmpty(directory) || !Directory.Exists(directory) || progress == null)
 			{
+				Logger?.Error($"Couldn't start loading obj dir: {directory}");
 				return;
 			}
 
@@ -324,7 +277,7 @@ public void LoadObjDirectory(string directory, IProgress<float>? progress, bool
 			else
 			{
 				Logger?.Info("Recreating index file");
-				CreateIndex(allFiles, progress); // do we need the array?
+				await CreateIndex(allFiles, progress); // do we need the array?
 				SerialiseHeaderIndexToFile(Settings.GetObjDataFullPath(Settings.IndexFileName), HeaderIndex, GetOptions());
 			}
 

AvaGui/Models/ObjectEditorModel.cs

@@ -0,0 +1,8 @@
+using OpenLoco.ObjectEditor.Data;
+using OpenLoco.ObjectEditor.Headers;
+using OpenLoco.ObjectEditor.Objects;
+
+namespace AvaGui.Models
+{
+	public record ObjectIndexModel(string Name, DatFileType DatFileType, ObjectType ObjectType, SourceGame SourceGame, uint32_t Checksum, VehicleType? VehicleType);
+}

AvaGui/Models/ObjectIndexModel.cs

@@ -9,7 +9,7 @@ namespace AvaGui.Models
 {
 	public class ObjectTypeToMaterialIconConverter : IValueConverter
 	{
-		public object? Convert(object value, Type targetType, object parameter, CultureInfo culture)
+		public object? Convert(object? value, Type targetType, object? parameter, CultureInfo culture)
 		{
 			if (Enum.TryParse<DatFileType>(value as string, out var datType) && DatTypeMapping.TryGetValue(datType, out var datIcon))
 			{

AvaGui/Models/ObjectTypeToMaterialIconConverter.cs

@@ -7,8 +7,13 @@ namespace AvaGui
 {
 	public class ViewLocator : IDataTemplate
 	{
-		public Control Build(object data)
+		public Control Build(object? data)
 		{
+			if (data == null)
+			{
+				return new TextBlock { Text = "<object passed in was null>" };
+			}
+
 			var name = data.GetType().FullName!.Replace("ViewModel", "View");
 			var type = Type.GetType(name);
 
@@ -20,7 +25,7 @@ public Control Build(object data)
 			return new TextBlock { Text = "Not Found: " + name };
 		}
 
-		public bool Match(object data)
+		public bool Match(object? data)
 			=> data is ViewModelBase;
 	}
 }

AvaGui/ViewLocator.cs

@@ -28,7 +28,7 @@ public class ObjectEditorViewModel : ReactiveObject, ILocoFileViewModel
 		public UiLocoFile? CurrentObject { private set; get; }
 
 		[Reactive]
-		public FileSystemItemBase CurrentFile { get; set; }
+		public FileSystemItemBase? CurrentFile { get; set; }
 
 		public ObjectEditorViewModel(ObjectEditorModel model)
 		{

AvaGui/ViewModels/DatTypes/ObjectEditorViewModel.cs

@@ -9,6 +9,8 @@
 using OpenLoco.ObjectEditor.Data;
 using ReactiveUI.Fody.Helpers;
 using System.Reactive;
+using System.Threading.Tasks;
+using System.Diagnostics;
 
 namespace AvaGui.ViewModels
 {
@@ -19,23 +21,28 @@ public class FolderTreeViewModel : ReactiveObject
 		public FolderTreeViewModel(ObjectEditorModel model)
 		{
 			Model = model;
+			Progress = new();
+			Progress.ProgressChanged += (a, b) =>
+			{
+				IndexingProgress = b;
+			};
 
-			RecreateIndex = ReactiveCommand.Create(() => LoadObjDirectory(CurrentDirectory, false));
+			RecreateIndex = ReactiveCommand.Create(async () => await LoadObjDirectoryAsync(CurrentDirectory, false));
 
 			_ = this.WhenAnyValue(o => o.CurrentDirectory)
-				.Subscribe(_ => LoadObjDirectory(CurrentDirectory, true));
+				.Subscribe(async _ => await LoadObjDirectoryAsync(CurrentDirectory, true));
 			_ = this.WhenAnyValue(o => o.DisplayVanillaOnly)
-				.Subscribe(_ => LoadObjDirectory(CurrentDirectory, true));
+				.Subscribe(async _ => await LoadObjDirectoryAsync(CurrentDirectory, true));
 			_ = this.WhenAnyValue(o => o.FilenameFilter)
 				.Throttle(TimeSpan.FromMilliseconds(500))
-				.Subscribe(_ => LoadObjDirectory(CurrentDirectory, true));
+				.Subscribe(async _ => await LoadObjDirectoryAsync(CurrentDirectory, true));
 			_ = this.WhenAnyValue(o => o.DirectoryItems)
 				.Subscribe(_ => this.RaisePropertyChanged(nameof(DirectoryFileCount)));
 
 			// loads the last-viewed folder
 			CurrentDirectory = Model.Settings.ObjDataDirectory;
 		}
-		public ReactiveCommand<Unit, Unit> RecreateIndex { get; }
+		public ReactiveCommand<Unit, Task> RecreateIndex { get; }
 
 		[Reactive]
 		public string CurrentDirectory { get; set; } = string.Empty;
@@ -50,36 +57,43 @@ public FolderTreeViewModel(ObjectEditorModel model)
 		public bool DisplayVanillaOnly { get; set; }
 
 		[Reactive]
-		public ObservableCollection<FileSystemItemBase> DirectoryItems { get; }
+		public ObservableCollection<FileSystemItemBase> DirectoryItems { get; private set; }
+
+		Progress<float> Progress { get; set; }
+
+		[Reactive]
+		public float IndexingProgress { get; set; }
 
-		private void LoadObjDirectory(string directory, bool useExistingIndex)
+		private async Task LoadObjDirectoryAsync(string directory, bool useExistingIndex)
 		{
-			DirectoryItems = new(LoadObjDirectoryCore(directory, useExistingIndex));
+			DirectoryItems = new(await LoadObjDirectoryCoreAsync(directory, useExistingIndex));
 
 			// really just for debugging - puts all dat file types in the collection, even if they don't have anything in them
 			//foreach (var dat in Enum.GetValues<DatFileType>().Except(DirectoryItems.Select(x => ((FileSystemDatGroup)x).DatFileType)))
 			//{
 			//	DirectoryItems.Add(new FileSystemDatGroup(string.Empty, dat, new ObservableCollection<FileSystemItemBase>()));
 			//}
 
-			IEnumerable<FileSystemItemBase> LoadObjDirectoryCore(string directory, bool useExistingIndex)
+			async Task<List<FileSystemItemBase>> LoadObjDirectoryCoreAsync(string directory, bool useExistingIndex)
 			{
+				var result = new List<FileSystemItemBase>();
+
 				if (string.IsNullOrEmpty(directory))
 				{
-					yield break;
+					return result;
 				}
 
 				var dirInfo = new DirectoryInfo(directory);
 
 				if (!dirInfo.Exists)
 				{
-					yield break;
+					return result;
 				}
 
 				// todo: load each file
 				// check if its object, scenario, save, landscape, g1, sfx, tutorial, etc
 
-				Model.LoadObjDirectory(directory, null, useExistingIndex);
+				await Model.LoadObjDirectoryAsync(directory, Progress, useExistingIndex);
 
 				var groupedDatObjects = Model.HeaderIndex
 					.Where(o => (string.IsNullOrEmpty(FilenameFilter) || o.Value.Name.Contains(FilenameFilter, StringComparison.CurrentCultureIgnoreCase)) && (!DisplayVanillaOnly || o.Value.SourceGame == SourceGame.Vanilla))
@@ -135,11 +149,13 @@ IEnumerable<FileSystemItemBase> LoadObjDirectoryCore(string directory, bool useE
 							subNodes));
 					}
 
-					yield return new FileSystemDatGroup(
+					result.Add(new FileSystemDatGroup(
 						string.Empty,
 						datObjGroup.Key,
-						groups);
+						groups));
 				}
+
+				return result;
 			}
 		}
 

AvaGui/ViewModels/FolderTreeViewModel.cs

@@ -1,4 +1,4 @@
-global using HeaderIndex = System.Collections.Generic.Dictionary<string, AvaGui.Models.IndexObjectHeader>;
+global using HeaderIndex = System.Collections.Generic.Dictionary<string, AvaGui.Models.ObjectIndexModel>;
 using Avalonia;
 using AvaGui.Models;
 using ReactiveUI;
@@ -62,10 +62,10 @@ public class MainWindowViewModel : ViewModelBase
 		public string WindowTitle => $"{ObjectEditorModel.ApplicationName} - {ApplicationVersion} ({LatestVersionText})";
 
 		[Reactive]
-		Version ApplicationVersion { get; }
+		public Version ApplicationVersion { get; set; }
 
 		[Reactive]
-		string LatestVersionText { get; } = "Up-to-date";
+		public string LatestVersionText { get; set; } = "Up-to-date";
 
 		public MainWindowViewModel()
 		{
@@ -190,7 +190,9 @@ public async Task SelectNewFolder()
 			var dirPath = dir.Path.LocalPath;
 			if (Directory.Exists(dirPath) && !Model.Settings.ObjDataDirectories.Contains(dirPath))
 			{
-				await Model.LoadObjDirectoryAsync(dirPath, null, false);
+				FolderTreeViewModel.CurrentDirectory = dirPath; // this will cause the reindexing
+																//var progress = new Progress<float>();
+																//await Model.LoadObjDirectoryAsync(dirPath, progress, false);
 				var menuItem = new MenuItemModel(
 					dirPath,
 					ReactiveCommand.Create(() => FolderTreeViewModel.CurrentDirectory = dirPath)