feat: add basic ulog file structure

src/Asv.ULog/Asv.ULog.csproj.DotSettings

@@ -3,6 +3,12 @@
     <s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=serializers/@EntryIndexedValue">True</s:Boolean>
     <s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=tokens/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=tokens_005Cfileheader/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=ulogfile_005Cmodels/@EntryIndexedValue">False</s:Boolean>
+	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=ulogfile_005Cprocessor/@EntryIndexedValue">False</s:Boolean>
+	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=ulogfile_005Cprocessors_005Cfileelements/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=ulogfile_005Cprocessors_005Chandlers/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=ulogfile_005Cprocessors_005Cmodels/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=ulogfile_005Cprocessor_005Chandlers/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=ulogvalue_005Ccontainer/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=ulogvalue_005Csimple/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=ulogvalue/@EntryIndexedValue">True</s:Boolean>

src/Asv.ULog/ULogFile/Processors/FileElements/CompositeKey.cs

@@ -0,0 +1,7 @@
+namespace Asv.ULog.ULogFile.Processors;
+
+public sealed class CompositeKey
+{
+    public required ULogTypeDefinition Type { get; init; }
+    public required string Name { get; init; }
+}

src/Asv.ULog/ULogFile/Processors/FileElements/LoggedStringMessage.cs

@@ -0,0 +1,10 @@
+namespace Asv.ULog.ULogFile.Processors;
+
+public sealed class LoggedStringMessage
+{
+    public required ULogLoggedStringMessageToken.ULogLevel LogLevel { get; init; }
+
+    public required TimeSpan Time { get; init; }
+
+    public required string Message { get; init; }
+}

src/Asv.ULog/ULogFile/Processors/FileElements/SubscriptionMessage.cs

@@ -0,0 +1,7 @@
+namespace Asv.ULog.ULogFile.Processors;
+
+public sealed class SubscriptionMessage
+{
+    public required byte MultiId { get; init; }
+    public required string MessageName { get; init; }
+}

src/Asv.ULog/ULogFile/Processors/FileElements/SynchronizationMessage.cs

@@ -0,0 +1,6 @@
+namespace Asv.ULog.ULogFile.Processors;
+
+public sealed class SynchronizationMessage
+{
+    public static byte[] SyncMagic { get; } = ULogSynchronizationMessageToken.SyncMagic;
+}

src/Asv.ULog/ULogFile/Processors/FileElements/TaggedLoggedStringMessage.cs

@@ -0,0 +1,10 @@
+namespace Asv.ULog.ULogFile.Processors;
+
+public sealed class TaggedLoggedStringMessage
+{
+    public required ULogTaggedLoggedStringMessageToken.ULogLevel LogLevel { get; init; }
+
+    public required TimeSpan Time { get; init; }
+
+    public required string Message { get; init; }
+}

src/Asv.ULog/ULogFile/Processors/Handlers/EmptyTokenHandler.cs

@@ -0,0 +1,6 @@
+namespace Asv.ULog.ULogFile.Processors;
+
+public class EmptyTokenHandler : ITokenHandler
+{
+    public void Handle(IULogToken token, ProcessorContext context) { }
+}

src/Asv.ULog/ULogFile/Processors/Handlers/FileHeaderTokenHandler.cs

@@ -0,0 +1,22 @@
+using System.Diagnostics;
+
+namespace Asv.ULog.ULogFile.Processors;
+
+public class FileHeaderTokenHandler : ITokenHandler
+{
+    public void Handle(IULogToken token, ProcessorContext context)
+    {
+        Debug.Assert(context.File.ReadState == TokenPlaceFlags.Header);
+        var header = token as ULogFileHeaderToken;
+
+        context.File.Time = FromTimeStampToDateTime(header!.Timestamp).TimeOfDay;
+        context.File.Version = header.Version;
+    }
+
+    private static DateTimeOffset FromTimeStampToDateTime(ulong timestamp)
+    {
+        var dateTimeOffset = new DateTimeOffset();
+        dateTimeOffset = dateTimeOffset.AddSeconds(timestamp);
+        return dateTimeOffset;
+    }
+}

src/Asv.ULog/ULogFile/Processors/Handlers/FlagBitsTokenHandler.cs

@@ -0,0 +1,15 @@
+using System.Diagnostics;
+
+namespace Asv.ULog.ULogFile.Processors;
+
+public class FlagBitsTokenHandler : ITokenHandler
+{
+    public void Handle(IULogToken token, ProcessorContext context)
+    {
+        Debug.Assert(context.File.ReadState == TokenPlaceFlags.Header);
+        var flagBits = token as ULogFlagBitsMessageToken;
+        context.File.Definition.FlagBits.CompatFlags = flagBits!.CompatFlags;
+        context.File.Definition.FlagBits.CompatFlags = flagBits.IncompatFlags;
+        context.File.Definition.FlagBits.AppendedOffsets = flagBits.AppendedOffsets;
+    }
+}

src/Asv.ULog/ULogFile/Processors/Handlers/ITokenHandler.cs

@@ -0,0 +1,6 @@
+namespace Asv.ULog.ULogFile.Processors;
+
+public interface ITokenHandler
+{
+    public void Handle(IULogToken token, ProcessorContext context);
+}

src/Asv.ULog/ULogFile/Processors/ProcessorContext.cs

@@ -0,0 +1,6 @@
+namespace Asv.ULog.ULogFile.Processors;
+
+public class ProcessorContext
+{
+    public required ULogFile File { get; init; }
+}

src/Asv.ULog/ULogFile/Processors/TokenProcessor.cs

@@ -0,0 +1,27 @@
+namespace Asv.ULog.ULogFile.Processors;
+
+public class TokenProcessor
+{
+    private readonly Dictionary<ULogToken, ITokenHandler> _handlers;
+    private readonly ProcessorContext _context;
+
+    public TokenProcessor(Dictionary<ULogToken, ITokenHandler> handlers, ProcessorContext context)
+    {
+        _handlers = handlers;
+        _context = context;
+    }
+
+    public TokenPlaceFlags Process(IULogToken token)
+    {
+        if (_handlers.TryGetValue(token.TokenType, out var handler))
+        {
+            handler.Handle(token, _context);
+            return token.TokenSection;
+        }
+        else
+        {
+            return _context.File.ReadState; // while not all tokens are handled
+            throw new UnknownTokenException();
+        }
+    }
+}

src/Asv.ULog/ULogFile/ULogFile.cs

@@ -0,0 +1,163 @@
+using System.Buffers;
+using System.Collections.Immutable;
+using System.Diagnostics;
+using Asv.ULog.ULogFile.Processors;
+
+namespace Asv.ULog.ULogFile;
+
+public class Data
+{
+    public IDictionary<ushort, SubscriptionMessage> SubscriptionMessages { get; } = new Dictionary<ushort, SubscriptionMessage>();
+    public IDictionary<ushort, ULogArray> LoggedDataMessages { get; } = new Dictionary<ushort, ULogArray>();
+    public ICollection<LoggedStringMessage> LoggedStringMessages { get; } = new List<LoggedStringMessage>();
+    public IDictionary<ushort, TaggedLoggedStringMessage> TaggedLoggedStringMessages { get; } = new Dictionary<ushort, TaggedLoggedStringMessage>();
+    public ICollection<SynchronizationMessage> SynchronizationMessages { get; } = new List<SynchronizationMessage>();
+    public ICollection<ushort> DropoutMessages { get; } = new List<ushort>();
+
+    public IDictionary<CompositeKey, string> InformationMessages { get; } = new Dictionary<CompositeKey, string>();
+    public IDictionary<CompositeKey, ICollection<string>> MultiInformationMessages { get; } = new Dictionary<CompositeKey, ICollection<string>>();
+    public IDictionary<CompositeKey, string> Parameters { get; } = new Dictionary<CompositeKey, string>();
+    public IDictionary<CompositeKey, string> DefaultParameters { get; } = new Dictionary<CompositeKey, string>();
+}
+
+public class Definition
+{
+    public ULogFlagBitsMessageToken FlagBits { get; } = new();
+    public ICollection<IDictionary<string, ULogValue>> FormatMessages { get; } = new List<IDictionary<string, ULogValue>>();
+    public IDictionary<CompositeKey, string> InformationMessages { get; } = new Dictionary<CompositeKey, string>();
+    public IDictionary<CompositeKey, ICollection<string>> MultiInformationMessages { get; } = new Dictionary<CompositeKey, ICollection<string>>();
+    public IDictionary<CompositeKey, string> Parameters { get; } = new Dictionary<CompositeKey, string>();
+    public IDictionary<CompositeKey, string> DefaultParameters { get; } = new Dictionary<CompositeKey, string>();
+}
+
+public class ULogFile
+{
+    private readonly IULogReader _reader;
+    private readonly TokenProcessor _processor;
+
+    /*
+        Header
+    */
+    public TimeSpan Time { get; set; }
+    public byte Version { get; set; }
+
+
+    public readonly Definition Definition = new();
+    public readonly Data Data = new();
+
+    public TokenPlaceFlags ReadState { get; private set; } = TokenPlaceFlags.None;
+
+    public ULogFile(IULogReader reader)
+    {
+        _reader = reader;
+        _processor = new TokenProcessor(new Dictionary<ULogToken, ITokenHandler>
+        {
+            { ULogToken.FileHeader, new FileHeaderTokenHandler() },
+            { ULogToken.FlagBits, new FlagBitsTokenHandler() },
+            { ULogToken.Unsubscription, new EmptyTokenHandler() } // cause unsub is not used currently
+        }, new ProcessorContext{ File = this });
+    }
+
+    public void Deserialize(ref ReadOnlySequence<byte> data)
+    {
+        ReadState = TokenPlaceFlags.Header;
+        var rdr = new SequenceReader<byte>(data);
+        while (_reader.TryRead(ref rdr, out var token))
+        {
+            Debug.Assert(token is not null);
+            ReadState = _processor.Process(token);
+        }
+    }
+
+    public void Serialize(ref Span<byte> buffer)
+    {
+        throw new NotImplementedException();
+    }
+
+    public static ULogValue Create(ULogLoggedDataMessageToken data,IReadOnlyDictionary<string, ULogFormatMessageToken> messages,
+        IReadOnlyDictionary<ushort, ULogSubscriptionMessageToken> subscriptions, out string messageName)
+    {
+        var sub = subscriptions[data.MessageId];
+        var message = messages[sub.MessageName];
+        messageName = sub.MessageName;
+        var obj = CreateReference(message.MessageName,messages);
+        var span = new ReadOnlySpan<byte>(data.Data);
+        obj.Deserialize(ref span);
+        return obj;
+    }
+
+    private static ULogObject CreateReference(string name, IReadOnlyDictionary<string, ULogFormatMessageToken> messages)
+    {
+        var message = messages[name];
+        var builder = ImmutableArray.CreateBuilder<ULogProperty>(message.Fields.Count);
+        foreach (var field in message.Fields)
+        {
+            var prop = Create(field.Type, messages);
+            builder.Add(new ULogProperty(field.Name,prop));
+        }
+        if (message.Fields[^1].Name.StartsWith("_padding"))
+        {
+            builder.RemoveAt(builder.Count - 1);
+        }
+        return new ULogObject(builder.ToImmutable());
+    }
+
+    private static ULogValue Create(ULogTypeDefinition fieldType, IReadOnlyDictionary<string, ULogFormatMessageToken> messages)
+    {
+        ULogValue value;
+        if (fieldType.BaseType == ULogType.ReferenceType)
+        {
+            value = CreateReference(fieldType.TypeName, messages);
+        }
+        else
+        {
+            value = CreateSimple(fieldType);
+        }
+
+        if (!fieldType.IsArray) return value;
+
+        var buidler = ImmutableArray.CreateBuilder<ULogValue>(fieldType.ArraySize);
+        buidler.Add(value);
+        for (var i = 1; i < fieldType.ArraySize; i++)
+        {
+            buidler.Add(value.CloneToken());    
+        }
+
+        return new ULogArray(buidler.ToImmutable());
+
+    }
+
+    private static ULogSimple CreateSimple(ULogTypeDefinition type)
+    {
+        switch (type.BaseType)
+        {
+            case ULogType.Int8:
+                return new ULogInt8();
+            case ULogType.UInt8:
+                return new ULogUInt8();
+            case ULogType.Int16:
+                return new ULogInt16();
+            case ULogType.UInt16:
+                return new ULogUInt16();
+            case ULogType.Int32:
+                return new ULogInt32();
+            case ULogType.UInt32:
+                return new ULogUInt32();
+            case ULogType.Int64:
+                return new ULogInt64();
+            case ULogType.UInt64:
+                return new ULogUInt64();
+            case ULogType.Float:
+                return new ULogFloat();
+            case ULogType.Double:
+                return new ULogDouble();
+            case ULogType.Bool:
+                return new ULogBool();
+            case ULogType.Char:
+                return new ULogChar();
+            case ULogType.ReferenceType:
+            default:
+                throw new ArgumentOutOfRangeException();
+        }
+    }
+}

src/Asv.ULog/ULogReader.cs

@@ -97,6 +97,8 @@ public bool TryRead(ref SequenceReader<byte> rdr, out IULogToken? token)
                     {
                         throw new WrongTokenSectionException();
                     }
+
+                    if (!InternalReadToken(ref rdr, ref token)) return false;
                 }
                 catch (ULogException)
                 {

src/Asv.Ulog.Tests/ULog/ULogFile.Tests.cs

@@ -0,0 +1,45 @@
+using System.Buffers;
+using Asv.ULog.ULogFile;
+using Xunit.Abstractions;
+
+namespace Asv.Ulog.Tests;
+
+public class ULogFileTests
+{
+    private readonly ITestOutputHelper _output;
+
+    public ULogFileTests(ITestOutputHelper output)
+    {
+        _output = output;
+    }
+
+    [Fact]
+    public void Deserialize_AllData_Success()
+    {
+        var reader = ULog.ULog.CreateReader();
+        var data = new ReadOnlySequence<byte>(TestData.ulog_log_small);
+        var uLogFile = new ULogFile(reader);
+        uLogFile.Deserialize(ref data);
+
+        _output.WriteLine("===============Header===============");
+        _output.WriteLine($"Version: {uLogFile.Version}");
+        _output.WriteLine($"Time: {uLogFile.Time}");
+        _output.WriteLine("===============Definition===============");
+        _output.WriteLine("===============Flag Bits===============");
+        _output.WriteLine("CompatFlags:");
+        foreach (var b in uLogFile.Definition.FlagBits.CompatFlags)
+        {
+            _output.WriteLine($"{b}");
+        }
+        _output.WriteLine("Incompat flags:");
+        foreach (var b in uLogFile.Definition.FlagBits.IncompatFlags)
+        {
+            _output.WriteLine($"{b}");
+        }
+        _output.WriteLine("Appended offsets:");
+        foreach (var b in uLogFile.Definition.FlagBits.AppendedOffsets)
+        {
+            _output.WriteLine($"{b}");
+        }
+    }
+}