Merge pull request #938 from jan-van-dieren/Nested-Object-Parsing

Extended matlab parser with structures, cell matrices and character matrices

Author: cdrnet

data/Matlab/cell-array-nested.mat

@@ -64,5 +64,6 @@ Control.Describe now includes CPU architecture and family identifier if know</Pa
             <PrivateAssets>all</PrivateAssets>
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
         </PackageReference>
+        <PackageReference Include="OneOf" Version="3.0.223" />
     </ItemGroup>
 </Project>

data/Matlab/struct-nested.mat

@@ -70,6 +70,18 @@ public static Matrix<T> Unpack<T>(MatlabMatrix matrixData)
             return Parser.ParseMatrix<T>(matrixData.Data);
         }
 
+        /// <summary>
+        /// Unpacks a MATLAB object that cannot be mapped to a MathNet.Numerics matrix. This could be any nesting of cell matrices and structures.
+        /// The nesting must bottom out on either a MathNet.Numerics matrix (if the lowest level values here are matrices of a numeric type) or a character mattrix (if the lowest level value is a matrix of a char type)
+        /// Since structures and cell matrices can have different data types in different fields and can even be nested the type of the field value cannot be more specific than a nested object
+        /// </summary>
+        /// <returns></returns>
+        public static NestedObject NonNumeric(MatlabMatrix structData)
+        {
+            return Parser.ParseNonNumeric(structData.Data);
+        }
+
+
         /// <summary>
         /// Read the first or a specific matrix from a MATLAB file stream.
         /// </summary>

src/Data.Matlab/Data.Matlab.csproj

@@ -0,0 +1,92 @@
+// <copyright file="MatlabStructure.cs" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// https://numerics.mathdotnet.com
+// https://github.com/mathnet/mathnet-numerics
+//
+// Copyright (c) 2009-$CURRENT_YEAR$ Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+using MathNet.Numerics.LinearAlgebra;
+using OneOf;
+using System.Collections.Generic;
+using System.Text;
+
+namespace MathNet.Numerics.Data.Matlab
+{
+    public class MatlabStructure : Dictionary<string, NestedObject>
+    {
+    }
+
+    public class MatlabCellMatrix
+    {
+        public NestedObject[,] Data { get; private set; }
+
+        public MatlabCellMatrix(int rows, int cols)
+        {
+            Data = new NestedObject[rows, cols];
+        }
+    }
+
+    public class MatlabCharMatrix
+    {
+        /// <summary>
+        /// Typically not that relevant, for UTF32 encoding however each single char is actually 2 chars (hence the string type)
+        /// </summary>
+        public Encoding Encoding { get; private set; }
+        public string[,] Data { get; private set; }
+
+        public MatlabCharMatrix(int rows, int cols, Encoding encoding)
+        {
+            Encoding = encoding;
+
+            Data = new string[rows, cols];
+        }
+
+        /// <summary>
+        /// Returns each row as a single string
+        /// </summary>
+        /// <returns></returns>
+        public string[] ConcatRows()
+        {
+            string[] result = new string[Data.GetLength(0)];
+            for (int col = 0; col < Data.GetLength(1); col++)
+            {
+                for (int row = 0; row < Data.GetLength(0); row++)
+
+                {
+                    result[row] += Data[row, col];
+                }
+            }
+
+            return result;
+        }
+    }
+
+    public class NestedObject : OneOfBase<MatlabStructure, MatlabCellMatrix, MatlabCharMatrix, MatlabMatrix>
+    {
+        public NestedObject(OneOf<MatlabStructure, MatlabCellMatrix, MatlabCharMatrix, MatlabMatrix> input) : base(input)
+        {
+        }
+    }
+}

src/Data.Matlab/MatlabReader.cs

@@ -141,51 +141,248 @@ internal static List<MatlabMatrix> ParseFile(Stream stream)
         internal static Matrix<T> ParseMatrix<T>(byte[] data)
             where T : struct, IEquatable<T>, IFormattable
         {
-            using (var stream = new MemoryStream(data))
-            using (var reader = new BinaryReader(stream))
+            Func<BinaryReader, ArrayClass, bool, int, int, Matrix<T>> parser = (BinaryReader r, ArrayClass a, bool complex, int rows, int columns) =>
             {
-                // Array Flags tag (8 bytes)
-                reader.BaseStream.Seek(8, SeekOrigin.Current);
+                // Data
+                switch (a)
+                {
+                    case ArrayClass.Sparse:
+                        return PopulateSparseMatrix<T>(r, complex, rows, columns);
+                    case ArrayClass.Function:
+                    case ArrayClass.Character:
+                    case ArrayClass.Object:
+                    case ArrayClass.Structure:
+                    case ArrayClass.Cell:
+                    case ArrayClass.Unknown:
+                        throw new NotSupportedException();
+                    default:
+                        return PopulateDenseMatrix<T>(r, complex, rows, columns);
+                }
+            };
 
-                // Array Flags data: flags (byte 3), class (byte 4) (8 bytes)
-                var arrayClass = (ArrayClass)reader.ReadByte();
-                var flags = reader.ReadByte();
-                var complex = (flags & (byte)ArrayFlags.Complex) == (byte)ArrayFlags.Complex;
-                reader.BaseStream.Seek(6, SeekOrigin.Current);
+            return ParseObject(data, parser);
+        }
 
-                // Dimensions Array tag (8 bytes)
-                reader.BaseStream.Seek(4, SeekOrigin.Current);
-                var numDimensions = reader.ReadInt32()/8;
-                if (numDimensions > 2)
+        /// <summary>
+        /// For parsing nayhting that cannot be mapped to a MathNet.Numerics matrix
+        /// </summary>
+        /// <param name="data"></param>
+        /// <returns></returns>
+        /// <exception cref="NotSupportedException"></exception>
+        internal static NestedObject ParseNonNumeric(byte[] data)
+        {
+            Func<BinaryReader, ArrayClass, bool, int, int, NestedObject> parser = (BinaryReader r, ArrayClass a, bool complex, int rows, int columns) =>
+            {
+                // Data
+                switch (a)
                 {
-                    throw new NotSupportedException("Only 1 and 2 dimensional arrays are supported.");
+                    case ArrayClass.Character:
+                        return PopulateCharacterMatrix(r, rows, columns);
+                    case ArrayClass.Structure:
+                        return PopulateStructure(r);
+                    case ArrayClass.Cell:
+                        return PopulateCellMatrix(r, complex, rows, columns);
+                    case ArrayClass.Unknown:
+                        throw new NotSupportedException();
+                    default:
+                        throw new NotSupportedException();
                 }
+            };
 
-                // Dimensions Array data: row and column count (8 bytes)
-                var rows = reader.ReadInt32();
-                var columns = reader.ReadInt32();
+            return ParseObject(data, parser);
+        }
 
-                // Array name
-                ReadElementTag(reader, out _, out var size, out var isSmallBlock);
-                reader.BaseStream.Seek(size, SeekOrigin.Current);
-                SkipElementPadding(reader, size, isSmallBlock);
+        private static NestedObject PopulateCharacterMatrix(BinaryReader reader, int rows, int columns)
+        {
+            ReadElementTag(reader, out var type, out var size, out var isSmallBlock);
+
+            MatlabCharMatrix result;
+            Encoding encoding;
+
+            switch (type)
+            {
+                case DataType.Utf8:
+                    encoding = Encoding.UTF8;
+                    break;
+                case DataType.Utf16:
+                    encoding = Encoding.Unicode;
+                    break;
+                case DataType.Utf32:
+                    encoding = Encoding.UTF32;
+                    break;
+                default:
+                    throw new NotImplementedException($"Could not parse char array due to unsupported encoding: {type}");
+            }
+
+            result = new MatlabCharMatrix(rows, columns, encoding);
+
+            for (int col = 0; col < columns; col++)
+            {
+                for (int row = 0; row < rows; row++)
+                {
+                    byte[] newChar;
+                    if(encoding.IsSingleByte)
+                    {
+                        newChar = reader.ReadBytes(1);
+                    }
+                    else
+                    {
+                        newChar = reader.ReadBytes(2);
+                    }
+
+                    result.Data[row, col] = encoding.GetString(newChar);
+                }
+            }
+
+            return new NestedObject(result);
+        }
+
+        internal static T ParseObject<T>(byte[] data, Func<BinaryReader, ArrayClass, bool,int,int,T> parser)
+        {
+            using (var stream = new MemoryStream(data))
+            using (var reader = new BinaryReader(stream))
+            {
+                (ArrayClass arrayClass,
+                bool complex,
+                int rows, int columns, _) = ParseObjectHeader(reader);
 
                 // Data
+                return parser(reader, arrayClass, complex, rows, columns);
+            }
+        }
+
+        /// <summary>
+        /// Reads the object header and skips any remaining padding
+        /// </summary>
+        /// <param name="reader"></param>
+        /// <returns></returns>
+        /// <exception cref="NotSupportedException"></exception>
+        private static (ArrayClass arrayClass, bool complex, int rows, int columns, string name) ParseObjectHeader(BinaryReader reader)
+        {
+            // Array Flags tag (8 bytes)
+            reader.BaseStream.Seek(8, SeekOrigin.Current);
+
+            // Array Flags data: flags (byte 3), class (byte 4) (8 bytes)
+            ArrayClass arrayClass = (ArrayClass)reader.ReadByte();
+            var flags = reader.ReadByte();
+            bool complex = (flags & (byte)ArrayFlags.Complex) == (byte)ArrayFlags.Complex;
+            reader.BaseStream.Seek(6, SeekOrigin.Current);
+
+            // Dimensions Array tag (8 bytes)
+            reader.BaseStream.Seek(4, SeekOrigin.Current);
+            var numDimensions = reader.ReadInt32() / 8;
+            if (numDimensions > 2)
+            {
+                throw new NotSupportedException("Only 1 and 2 dimensional arrays are supported.");
+            }
+
+            // Dimensions Array data: row and column count (8 bytes)
+            int rows = reader.ReadInt32();
+            int columns = reader.ReadInt32();
+
+            // Array name
+            ReadElementTag(reader, out _, out var size, out var isSmallBlock);
+            byte[] nameBytes = new byte[size];
+            reader.BaseStream.Read(nameBytes,0, size);
+            string name = Encoding.UTF8.GetString(nameBytes);
+
+            SkipElementPadding(reader, size, isSmallBlock);
+
+            return (arrayClass, complex, rows, columns, name);
+        }
+
+        private static NestedObject PopulateStructure(BinaryReader reader)
+        {
+            // after the common fields for all arrays a structure has the length for the field names as a short data element
+            // acording to the docs MATLAB always sets this to 32 bytes (31 chars + NULL) so we don't actually need to check it
+            ReadElementTag(reader, out _, out _, out _);
+
+            int nameLength = reader.ReadInt32();
+
+            // field names are saved as an miINT8 data element
+            // each name is padded to align on 32 bytes and NULL terminated
+            ReadElementTag(reader, out _, out var size, out var isSmallBlock);
+
+            List<string> fieldNames = new List<string>();
+            int bytesRead = 0;
+
+            while(bytesRead < size)
+            {
+                byte[] currentName = reader.ReadBytes(nameLength);
+                fieldNames.Add(Encoding.UTF8.GetString(currentName).TrimEnd((char)0));
+                bytesRead += nameLength;
+            }
+
+            SkipElementPadding(reader, size, isSmallBlock);
+
+            // each field of the structure could be any type supported by a matlab file
+            MatlabStructure result = new MatlabStructure();
+
+            for (int i = 0; i<fieldNames.Count; i++)
+            {
+                // to use the regular array parsing methods we need to know how much data to give them
+                ReadElementTag(reader, out _, out var fieldSize, out _);
+
+                // we also need to know what the array class is (maybe a nested structure or a cell)
+                (ArrayClass arrayClass, _, _, _, string name) = ParseObjectHeader(reader);
+
+                // reset reader back to expected position for further parsers
+                // the header has array flags (16 bytes), dimensions array (16 bytes) and array name (8 bytes)
+                reader.BaseStream.Seek(-40, SeekOrigin.Current);
+
+                byte[] arrayData = reader.ReadBytes(fieldSize);
+
                 switch (arrayClass)
                 {
-                    case ArrayClass.Sparse:
-                        return PopulateSparseMatrix<T>(reader, complex, rows, columns);
-                    case ArrayClass.Function:
-                    case ArrayClass.Character:
-                    case ArrayClass.Object:
                     case ArrayClass.Structure:
                     case ArrayClass.Cell:
-                    case ArrayClass.Unknown:
-                        throw new NotSupportedException();
+                    case ArrayClass.Character:
+                        result.Add(fieldNames[i], ParseNonNumeric(arrayData));
+                        break;
                     default:
-                        return PopulateDenseMatrix<T>(reader, complex, rows, columns);
+                        result.Add(fieldNames[i], new NestedObject(new MatlabMatrix(name, arrayData)));
+                        break;
                 }
             }
+
+            return new NestedObject(result);
+        }
+
+        private static NestedObject PopulateCellMatrix(BinaryReader reader, bool complex, int rows, int columns)
+        {
+            MatlabCellMatrix result = new MatlabCellMatrix(rows, columns);
+
+            for(int col = 0; col<columns; col++)
+            {
+                for(int row = 0; row<rows; row++)
+                {
+                    // to use the regular array parsing methods we need to know how much data to give them
+                    ReadElementTag(reader, out _, out var fieldSize, out _);
+
+                    // we also need to know what the array class is (maybe a nested structure or a cell)
+                    (ArrayClass arrayClass, _, _, _, string name) = ParseObjectHeader(reader);
+
+                    // reset reader back to expected position for further parsers
+                    // the header has array flags (16 bytes), dimensions array (16 bytes) and array name (8 bytes)
+                    reader.BaseStream.Seek(-40, SeekOrigin.Current);
+
+                    byte[] arrayData = reader.ReadBytes(fieldSize);
+
+                    switch (arrayClass)
+                    {
+                        case ArrayClass.Structure:
+                        case ArrayClass.Cell:
+                        case ArrayClass.Character:
+                            result.Data[row, col] = ParseNonNumeric(arrayData);
+                            break;
+                        default:
+                            result.Data[row, col]  = new NestedObject(new MatlabMatrix(name, arrayData));
+                            break;
+                    }
+                }
+            }
+
+            return new NestedObject(result);
         }
 
         /// <summary>