add start of encoding

Author: LeftofZen

Dat/FileParsing/SawyerStreamReader.cs

@@ -502,14 +502,15 @@ public static ILocoStruct GetLocoStruct(ObjectType objectType, ReadOnlySpan<byte
 			};
 
 		// taken from openloco's SawyerStreamReader::readChunk
-		public static byte[] Decode(SawyerEncoding encoding, ReadOnlySpan<byte> data, int minDecodedBytes = int.MaxValue) => encoding switch
-		{
-			SawyerEncoding.Uncompressed => data.ToArray(),
-			SawyerEncoding.RunLengthSingle => DecodeRunLengthSingle(data, minDecodedBytes),
-			SawyerEncoding.RunLengthMulti => DecodeRunLengthMulti(DecodeRunLengthSingle(data)),
-			SawyerEncoding.Rotate => DecodeRotate(data),
-			_ => throw new InvalidDataException("Unknown chunk encoding scheme"),
-		};
+		public static byte[] Decode(SawyerEncoding encoding, ReadOnlySpan<byte> data, int minDecodedBytes = int.MaxValue)
+			=> encoding switch
+			{
+				SawyerEncoding.Uncompressed => data.ToArray(),
+				SawyerEncoding.RunLengthSingle => DecodeRunLengthSingle(data, minDecodedBytes),
+				SawyerEncoding.RunLengthMulti => DecodeRunLengthMulti(DecodeRunLengthSingle(data)),
+				SawyerEncoding.Rotate => DecodeRotate(data),
+				_ => throw new InvalidDataException("Unknown chunk encoding scheme"),
+			};
 
 		public static (RiffWavHeader header, byte[] data) LoadWavFile(string filename)
 			=> LoadWavFile(File.ReadAllBytes(filename));
@@ -578,7 +579,7 @@ public static ReadOnlySpan<byte> ReadChunkCore(ref ReadOnlySpan<byte> data)
 		// taken from openloco SawyerStreamReader::decodeRunLengthSingle
 		static byte[] DecodeRunLengthSingle(ReadOnlySpan<byte> data, int minDecodedBytes = int.MaxValue)
 		{
-			var ms = new MemoryStream();
+			using var ms = new MemoryStream();
 
 			for (var i = 0; i < data.Length; ++i)
 			{

Dat/FileParsing/SawyerStreamWriter.cs

@@ -98,13 +98,13 @@ public static void ExportMusicAsWave(string filename, RiffWavHeader header, byte
 			}
 		}
 
-		public static void Save(string filename, string objName, SourceGame sourceGame, ILocoObject locoObject, ILogger logger, bool allowWritingAsVanilla)
+		public static void Save(string filename, string objName, SourceGame sourceGame, SawyerEncoding encoding, ILocoObject locoObject, ILogger logger, bool allowWritingAsVanilla)
 		{
 			ArgumentNullException.ThrowIfNull(locoObject);
 
 			logger.Info($"Writing \"{objName}\" to {filename}");
 
-			var objBytes = WriteLocoObject(objName, sourceGame, logger, locoObject, allowWritingAsVanilla);
+			var objBytes = WriteLocoObject(objName, sourceGame, encoding, logger, locoObject, allowWritingAsVanilla);
 
 			try
 			{
@@ -123,43 +123,110 @@ public static void Save(string filename, string objName, SourceGame sourceGame,
 			logger.Info($"{objName} successfully saved to {filename}");
 		}
 
-		public static ReadOnlySpan<byte> WriteLocoObject(string objName, SourceGame sourceGame, ILogger logger, ILocoObject obj, bool allowWritingAsVanilla)
-		 => WriteLocoObjectStream(objName, sourceGame, logger, obj, allowWritingAsVanilla).ToArray();
+		public static ReadOnlySpan<byte> WriteLocoObject(string objName, SourceGame sourceGame, SawyerEncoding encoding, ILogger logger, ILocoObject obj, bool allowWritingAsVanilla)
+			=> WriteLocoObjectStream(objName, sourceGame, encoding, logger, obj, allowWritingAsVanilla).ToArray();
+		public static byte[] Encode(SawyerEncoding encoding, ReadOnlySpan<byte> data)
+			=> data.ToArray();
 
-		public static MemoryStream WriteLocoObjectStream(string objName, SourceGame sourceGame, ILogger logger, ILocoObject obj, bool allowWritingAsVanilla)
+		//public static byte[] Encode(SawyerEncoding encoding, ReadOnlySpan<byte> data)
+		//	=> encoding switch
+		//	{
+		//		SawyerEncoding.Uncompressed => data.ToArray(),
+		//		SawyerEncoding.RunLengthSingle => EncodeRunLengthSingle(data),
+		//		SawyerEncoding.RunLengthMulti => throw new NotImplementedException(),
+		//		SawyerEncoding.Rotate => throw new NotImplementedException(),
+		//		_ => throw new InvalidDataException("Unknown chunk encoding scheme"),
+		//	};
+
+		public static byte[] EncodeRunLengthSingle(ReadOnlySpan<byte> data)
 		{
-			using var objStream = new MemoryStream();
+			using var buffer = new MemoryStream();
+
+			var src = 0;
+			var srcEnd = data.Length;
+			var srcNormStart = 0;
+			byte count = 0;
+
+			while (src < srcEnd - 1)
+			{
+				if ((count != 0 && data[src] == data[src + 1]) || count > 125)
+				{
+					buffer.WriteByte((byte)(count - 1));
+					buffer.Write(data[srcNormStart..(srcNormStart + count)]);
+					srcNormStart += count;
+					count = 0;
+				}
+				if (data[src] == data[src + 1])
+				{
+					for (; count < 125 && src + count < srcEnd; count++)
+					{
+						if (data[src] != data[src + count])
+						{
+							break;
+						}
+					}
+					buffer.WriteByte((byte)(257 - count));
+					buffer.WriteByte(data[src]);
+					src += count;
+					srcNormStart = src;
+					count = 0;
+				}
+				else
+				{
+					count++;
+					src++;
+				}
+			}
+			if (src == srcEnd - 1)
+			{
+				count++;
+			}
+			if (count != 0)
+			{
+				buffer.WriteByte((byte)(count - 1));
+				buffer.Write(data[srcNormStart..(srcNormStart + count)]);
+			}
+
+			return buffer.ToArray();
+		}
+
+		public static MemoryStream WriteLocoObjectStream(string objName, SourceGame sourceGame, SawyerEncoding encoding, ILogger logger, ILocoObject obj, bool allowWritingAsVanilla)
+		{
+			using var rawObjStream = new MemoryStream();
 
 			// obj
 			var objBytes = ByteWriter.WriteLocoStruct(obj.Object);
-			objStream.Write(objBytes);
+			rawObjStream.Write(objBytes);
 
 			// string table
 			foreach (var ste in obj.StringTable.Table)
 			{
 				foreach (var language in ste.Value.Where(str => !string.IsNullOrEmpty(str.Value))) // skip strings with empty content
 				{
-					objStream.WriteByte((byte)language.Key);
+					rawObjStream.WriteByte((byte)language.Key);
 
 					var strBytes = Encoding.Latin1.GetBytes(language.Value);
-					objStream.Write(strBytes, 0, strBytes.Length);
-					objStream.WriteByte((byte)'\0');
+					rawObjStream.Write(strBytes, 0, strBytes.Length);
+					rawObjStream.WriteByte((byte)'\0');
 				}
 
-				objStream.WriteByte(0xff);
+				rawObjStream.WriteByte(0xff);
 			}
 
 			// variable data
 			if (obj.Object is ILocoStructVariableData objV)
 			{
 				var variableBytes = objV.Save();
-				objStream.Write(variableBytes);
+				rawObjStream.Write(variableBytes);
 			}
 
 			// graphics data
-			SaveImageTable(obj.G1Elements, objStream);
+			SaveImageTable(obj.G1Elements, rawObjStream);
+
+			rawObjStream.Flush();
 
-			objStream.Flush();
+			//todo: actually use encoding
+			using var encodedObjStream = new MemoryStream(Encode(encoding, rawObjStream.ToArray()));
 
 			// now obj is written, we can calculate the few bits of metadata (checksum and length) for the headers
 
@@ -181,9 +248,9 @@ public static MemoryStream WriteLocoObjectStream(string objName, SourceGame sour
 			// calculate checksum
 			var headerFlag = BitConverter.GetBytes(s5Header.Flags).AsSpan()[0..1];
 			var asciiName = objName.PadRight(8, ' ').Take(8).Select(c => (byte)c).ToArray();
-			s5Header.Checksum = SawyerStreamUtils.ComputeObjectChecksum(headerFlag, asciiName, objStream.ToArray());
+			s5Header.Checksum = SawyerStreamUtils.ComputeObjectChecksum(headerFlag, asciiName, encodedObjStream.ToArray());
 
-			var objHeader = new ObjectHeader(SawyerEncoding.Uncompressed, (uint32_t)objStream.Length);
+			var objHeader = new ObjectHeader(SawyerEncoding.Uncompressed, (uint32_t)encodedObjStream.Length);
 
 			// actual writing
 			var headerStream = new MemoryStream();
@@ -195,13 +262,13 @@ public static MemoryStream WriteLocoObjectStream(string objName, SourceGame sour
 			headerStream.Write(objHeader.Write());
 
 			// loco object itself, including string and graphics table
-			headerStream.Write(objStream.ToArray());
+			headerStream.Write(encodedObjStream.ToArray());
 
 			// stream cleanup
 			headerStream.Flush();
 
 			headerStream.Close();
-			objStream.Close();
+			encodedObjStream.Close();
 
 			return headerStream;
 		}

Gui/ViewModels/DatTypes/DatObjectEditorViewModel.cs

@@ -1,5 +1,6 @@
 using OpenLoco.Common.Logging;
 using OpenLoco.Dat;
+using OpenLoco.Dat.Data;
 using OpenLoco.Dat.FileParsing;
 using OpenLoco.Dat.Objects.Sound;
 using OpenLoco.Dat.Types;
@@ -181,6 +182,7 @@ void SaveCore(string filename)
 			SawyerStreamWriter.Save(filename,
 				S5HeaderViewModel?.Name ?? CurrentObject.DatFileInfo.S5Header.Name,
 				S5HeaderViewModel?.SourceGame ?? CurrentObject.DatFileInfo.S5Header.SourceGame,
+				SawyerEncoding.Uncompressed, // todo: change based on what user selected
 				CurrentObject.LocoObject,
 				Logger,
 				Model.Settings.AllowSavingAsVanillaObject);

Tests/LoadSaveTests.cs

@@ -17,71 +17,55 @@ public class LoadSaveTests
 		// TODO: find a way to not have to hardcode a path here (but this may be impossible as it will depend on a user's PC and Loco install path)
 		// TODO: find a nicer (and more automated) way to check Name+Image fields, StringTable and G1Table
 
-		static (ILocoObject, T) LoadObject<T>(string filename) where T : ILocoStruct
-		{
-			filename = Path.Combine(BaseObjDataPath, filename);
-			var fileSize = new FileInfo(filename).Length;
-			var logger = new Logger();
-			var loaded = SawyerStreamReader.LoadFullObjectFromFile(filename, logger);
+		static (DatFileInfo, ILocoObject, T) LoadObject<T>(string filename) where T : ILocoStruct
+			=> LoadObject<T>(File.ReadAllBytes(Path.Combine(BaseObjDataPath, filename)));
 
-			Assert.That(loaded, Is.Not.Null);
-			var (datFileInfo, locoObject) = loaded.Value;
-
-			Assert.Multiple(() =>
-			{
-				Assert.That(datFileInfo.S5Header.Checksum, Is.EqualTo(OriginalObjectFiles.Names[datFileInfo.S5Header.Name].SteamChecksum));
-				Assert.That(locoObject, Is.Not.Null);
-				Assert.That(datFileInfo!.ObjectHeader.DataLength, Is.EqualTo(fileSize - S5Header.StructLength - ObjectHeader.StructLength), "ObjectHeader.Length didn't match actual size of struct");
-			});
-
-			return (locoObject!, (T)locoObject!.Object);
-		}
-
-		static (ILocoObject, T) LoadObject<T>(ReadOnlySpan<byte> data) where T : ILocoStruct
+		static (DatFileInfo, ILocoObject, T) LoadObject<T>(ReadOnlySpan<byte> data) where T : ILocoStruct
 		{
 			var logger = new Logger();
-			var (datFileInfo, locoObject) = SawyerStreamReader.LoadFullObjectFromStream(data, logger: logger);
+			var (datFileInfo, locoObject) = SawyerStreamReader.LoadFullObjectFromStream(data, logger);
 
 #pragma warning disable IDE0079 // Remove unnecessary suppression
 #pragma warning disable NUnit2045 // Use Assert.Multiple - cannot use a ReadOnlySpan inside an anonymous method
+			Assert.That(datFileInfo.S5Header.Checksum, Is.EqualTo(OriginalObjectFiles.Names[datFileInfo.S5Header.Name].SteamChecksum));
 			Assert.That(locoObject, Is.Not.Null);
 #pragma warning restore NUnit2045 // Use Assert.Multiple
 #pragma warning restore IDE0079 // Remove unnecessary suppression
 			Assert.That(datFileInfo.ObjectHeader.DataLength, Is.EqualTo(data.Length - S5Header.StructLength - ObjectHeader.StructLength), "ObjectHeader.Length didn't match actual size of struct");
 
-			return (locoObject!, (T)locoObject!.Object);
+			return (datFileInfo, locoObject!, (T)locoObject!.Object);
 		}
 
 		static void LoadSaveGenericTest<T>(string filename, Action<ILocoObject, T> assertFunc) where T : ILocoStruct
 		{
-			var (obj1, struc1) = LoadObject<T>(filename);
+			var (datInfo1, obj1, struc1) = LoadObject<T>(filename);
 			assertFunc(obj1, struc1);
 
 			var logger = new Logger();
 			var objectName = filename.Split('.')[0];
-			var bytes1 = SawyerStreamWriter.WriteLocoObject(objectName, SourceGame.Vanilla, logger, obj1, false);
+			var bytes1 = SawyerStreamWriter.WriteLocoObject(objectName, SourceGame.Vanilla, datInfo1.ObjectHeader.Encoding, logger, obj1, false);
 
-			var (obj2, struc2) = LoadObject<T>(bytes1);
+			var (datInfo2, obj2, struc2) = LoadObject<T>(bytes1);
 			assertFunc(obj2, struc2);
 
-			var bytes2 = SawyerStreamWriter.WriteLocoObject(objectName, SourceGame.Vanilla, logger, obj2, false);
+			var bytes2 = SawyerStreamWriter.WriteLocoObject(objectName, SourceGame.Vanilla, datInfo2.ObjectHeader.Encoding, logger, obj2, false);
 
 			// we could just simply compare byte arrays and be done, but i wanted something that makes it easier to diagnose problems
 
 			// grab headers first
 			var s5Header1 = S5Header.Read(bytes1[0..S5Header.StructLength]);
 			var s5Header2 = S5Header.Read(bytes2[0..S5Header.StructLength]);
+			AssertS5Headers(s5Header1, s5Header2);
 
 			var objHeader1 = ObjectHeader.Read(bytes1[S5Header.StructLength..(S5Header.StructLength + ObjectHeader.StructLength)]);
 			var objHeader2 = ObjectHeader.Read(bytes2[S5Header.StructLength..(S5Header.StructLength + ObjectHeader.StructLength)]);
+			AssertObjHeaders(objHeader1, objHeader2);
 
 			// then grab object bytes
 			var bytes1ObjArr = bytes1[21..].ToArray();
 			var bytes2ObjArr = bytes2[21..].ToArray();
-
-			AssertS5Headers(s5Header1, s5Header2);
-			AssertObjHeaders(objHeader1, objHeader2);
 			Assert.That(bytes1ObjArr.ToArray(), Is.EqualTo(bytes2ObjArr.ToArray()));
+
 		}
 
 		static void AssertS5Headers(S5Header expected, S5Header actual)
@@ -99,6 +83,36 @@ static void AssertObjHeaders(ObjectHeader expected, ObjectHeader actual)
 					Assert.That(actual.DataLength, Is.EqualTo(expected.DataLength));
 				});
 
+		[TestCase]
+		public void TestEncoding()
+		{
+			var logger = new Logger();
+
+			//var filename = "OGFOWL.dat";
+			//var path = "C:\\Users\\bigba\\source\\repos\\OpenLoco\\OpenGraphics\\objects\\Vehicle\\Train\\4F";
+			//var fullData = File.ReadAllBytes(Path.Combine(path, filename));
+
+			var filename = "707.dat";
+			var fullData = File.ReadAllBytes(Path.Combine(BaseObjDataPath, filename));
+
+			if (!SawyerStreamReader.TryGetHeadersFromBytes(fullData, out var hdrs, logger))
+			{
+				Assert.Fail();
+				return;
+			}
+
+			var remainingData = fullData[(S5Header.StructLength + ObjectHeader.StructLength)..];
+
+			var decoded = SawyerStreamReader.Decode(hdrs.Obj.Encoding, remainingData);
+			var encoded = SawyerStreamWriter.Encode(hdrs.Obj.Encoding, decoded);
+			var decoded2 = SawyerStreamReader.Decode(hdrs.Obj.Encoding, encoded);
+			var encoded2 = SawyerStreamWriter.Encode(hdrs.Obj.Encoding, decoded2);
+
+			Assert.That(remainingData, Is.EqualTo(encoded));
+			Assert.That(decoded, Is.EqualTo(decoded2));
+			Assert.That(encoded, Is.EqualTo(encoded2));
+		}
+
 		[TestCase("AIRPORT1.DAT")]
 		public void AirportObject(string objectName)
 		{
@@ -656,7 +670,7 @@ void assertFunc(ILocoObject obj, RoadStationObject struc) => Assert.Multiple(()
 				Assert.That(struc.NumCompatible, Is.EqualTo(0), nameof(struc.NumCompatible));
 				Assert.That(struc.ObsoleteYear, Is.EqualTo(1945), nameof(struc.ObsoleteYear));
 				Assert.That(struc.PaintStyle, Is.EqualTo(0), nameof(struc.PaintStyle));
-				Assert.That(struc.RoadPieces, Is.EqualTo(0), nameof(struc.RoadPieces));
+				Assert.That(struc.RoadPieces, Is.EqualTo(RoadTraitFlags.None), nameof(struc.RoadPieces));
 				Assert.That(struc.SellCostFactor, Is.EqualTo(-17), nameof(struc.SellCostFactor));
 			});
 			LoadSaveGenericTest<RoadStationObject>(objectName, assertFunc);

Tests/ViewModelTests.cs

@@ -1,12 +1,10 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
+using NUnit.Framework;
 
 namespace Tests
 {
-	class ViewModelTests
+	[TestFixture]
+	public class ViewModelTests
 	{
+
 	}
 }