Add support for loading mxOPAQUE_CLASS types

* MAT_subsys.jl: New file MAT_subsys with methods to set, parse and retrieve subsystem data
* MAT_v5.jl: New method "read_opaque" to handle mxOPAQUE_CLASS
* MAT_v5.jl: New method "read_subsystem" to handle subsystem data
* MAT.jl: Update to clear subsystem cache after load

Author: foreverallama

src/MAT.jl

@@ -26,11 +26,12 @@ module MAT
 
 using HDF5, SparseArrays
 
+include("MAT_subsys.jl")
 include("MAT_HDF5.jl")
 include("MAT_v5.jl")
 include("MAT_v4.jl")
 
-using .MAT_HDF5, .MAT_v5, .MAT_v4
+using .MAT_HDF5, .MAT_v5, .MAT_v4, .MAT_subsys
 
 export matopen, matread, matwrite, @read, @write
 
@@ -133,6 +134,7 @@ function matread(filename::AbstractString)
     try
         vars = read(file)
     finally
+        MAT_subsys.clear_subsys!()
         close(file)
     end
     vars
@@ -165,7 +167,7 @@ function matwrite(filename::AbstractString, dict::AbstractDict{S, T}; compress::
         end
 
     else
-        
+
         file = matopen(filename, "w"; compress = compress)
         try
             for (k, v) in dict

src/MAT_subsys.jl

@@ -0,0 +1,226 @@
+module MAT_subsys
+
+const FWRAP_VERSION = 4
+
+mutable struct Subsys
+    num_names::UInt32
+    mcos_names::Vector{String}
+    class_id_metadata::Vector{UInt32}
+    object_id_metadata::Vector{UInt32}
+    saveobj_prop_metadata::Vector{UInt32}
+    obj_prop_metadata::Vector{UInt32}
+    dynprop_metadata::Vector{UInt32}
+    _u6_metadata::Vector{UInt32}
+    _u7_metadata::Vector{UInt32}
+    prop_vals_saved::Vector{Any}
+    _c3::Any
+    _c2::Any
+    prop_vals_defaults::Any
+    handle_data::Any
+    java_data::Any
+
+    Subsys() = new(
+        UInt32(0),
+        String[],
+        UInt32[],
+        UInt32[],
+        UInt32[],
+        UInt32[],
+        UInt32[],
+        UInt32[],
+        UInt32[],
+        Any[],
+        nothing,
+        nothing,
+        nothing,
+        nothing,
+        nothing
+    )
+end
+
+const subsys_cache = Ref{Union{Nothing,Subsys}}(nothing)
+
+function clear_subsys!()
+    subsys_cache[] = nothing
+end
+
+function load_subsys!(subsystem_data::Dict{String,Any}, swap_bytes::Bool)
+    subsys_cache[] = Subsys()
+    subsys_cache[].handle_data = get(subsystem_data, "handle", nothing)
+    subsys_cache[].java_data = get(subsystem_data, "java", nothing)
+    mcos_data = get(subsystem_data, "MCOS", nothing)
+    if mcos_data === nothing
+        return
+    end
+
+    fwrap_metadata = vec(mcos_data[2][1, 1])
+
+    # FIXME: Is this the best way to read?
+    # Integers are written as uint8 (with swap), interpret as uint32
+    version = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[1:4]) : fwrap_metadata[1:4])[1]
+    if version > FWRAP_VERSION
+        error("Unsupported FileWrapper version: $version")
+    end
+
+    subsys_cache[].num_names = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[5:8]) : fwrap_metadata[5:8])[1]
+    region_offsets = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[9:40]) : fwrap_metadata[9:40])
+
+    # Class and Property Names stored as list of null-terminated strings
+    start = 41
+    pos = start
+    name_count = 0
+    while name_count < subsys_cache[].num_names
+        if fwrap_metadata[pos] == 0x00
+            push!(subsys_cache[].mcos_names, String(fwrap_metadata[start:pos-1]))
+            name_count += 1
+            start = pos + 1
+            if name_count == subsys_cache[].num_names
+                break
+            end
+        end
+        pos += 1
+    end
+
+    subsys_cache[].class_id_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[1]+1:region_offsets[2]]) : fwrap_metadata[region_offsets[1]+1:region_offsets[2]])
+    subsys_cache[].saveobj_prop_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[2]+1:region_offsets[3]]) : fwrap_metadata[region_offsets[2]+1:region_offsets[3]])
+    subsys_cache[].object_id_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[3]+1:region_offsets[4]]) : fwrap_metadata[region_offsets[3]+1:region_offsets[4]])
+    subsys_cache[].obj_prop_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[4]+1:region_offsets[5]]) : fwrap_metadata[region_offsets[4]+1:region_offsets[5]])
+    subsys_cache[].dynprop_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[5]+1:region_offsets[6]]) : fwrap_metadata[region_offsets[5]+1:region_offsets[6]])
+    subsys_cache[]._u6_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[6]+1:region_offsets[7]]) : fwrap_metadata[region_offsets[6]+1:region_offsets[7]])
+    subsys_cache[]._u7_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[7]+1:region_offsets[8]]) : fwrap_metadata[region_offsets[7]+1:region_offsets[8]])
+
+    if version < 4
+        subsys_cache[].prop_vals_saved = mcos_data[2][3:end-2, 1]
+    else
+        subsys_cache[].prop_vals_saved = mcos_data[2][3:end-3, 1]
+        subsys_cache[]._c3 = mcos_data[2][end-2, 1]
+    end
+
+    subsys_cache[]._c2 = mcos_data[2][end-1, 1]
+    subsys_cache[].prop_vals_defaults = mcos_data[2][end, 1]
+end
+
+function get_classname(class_id::UInt32)
+    namespace_idx = subsys_cache[].class_id_metadata[class_id*4+1]
+    classname_idx = subsys_cache[].class_id_metadata[class_id*4+2]
+
+    namespace = if namespace_idx == 0
+        ""
+    else
+        subsys_cache[].mcos_names[namespace_idx-1] * "."
+    end
+
+    classname = namespace * subsys_cache[].mcos_names[classname_idx-1]
+    return classname
+end
+
+function get_object_metadata(object_id::UInt32)
+    return subsys_cache[].object_id_metadata[object_id*6+1:object_id*6+6]
+end
+
+function get_default_properties(class_id::UInt32)
+    # FIXME Should we use deepcopy here
+    return copy(subsys_cache[].prop_vals_defaults[class_id+1, 1])
+end
+
+function get_property_idxs(obj_type_id::UInt32, saveobj_ret_type::Bool)
+    prop_field_idxs = saveobj_ret_type ? subsys_cache[].saveobj_prop_metadata : subsys_cache[].obj_prop_metadata
+    nfields = 3
+    offset = 1
+    while obj_type_id > 0
+        nprops = prop_field_idxs[offset]
+        offset += 1 + (nfields * nprops)
+        offset += (offset + 1) % 2  # Padding
+        obj_type_id -= 1
+    end
+    nprops = prop_field_idxs[offset]
+    offset += 1
+    return prop_field_idxs[offset:offset+nprops*nfields-1]
+end
+
+function get_saved_properties(obj_type_id::UInt32, saveobj_ret_type::Bool)
+    save_prop_map = Dict{String,Any}()
+    prop_field_idxs = get_property_idxs(obj_type_id, saveobj_ret_type)
+    nprops = length(prop_field_idxs) ÷ 3
+    for i in 0:nprops-1
+        prop_name = subsys_cache[].mcos_names[prop_field_idxs[i*3+1]]
+        prop_type = prop_field_idxs[i*3+2]
+        if prop_type == 0
+            prop_value = subsys_cache[].mcos_names[prop_field_idxs[i*3+3]]
+        elseif prop_type == 1
+            # FIXME: Search for nested objects
+            prop_value = subsys_cache[].prop_vals_saved[prop_field_idxs[i*3+3]+1]
+        elseif prop_type == 2
+            prop_value = prop_field_idxs[i*3+3]
+        else
+            error("Unknown property type ID: $prop_type encountered during deserialization")
+        end
+        save_prop_map[prop_name] = prop_value
+    end
+    return save_prop_map
+end
+
+function get_properties(object_id::UInt32)
+    if object_id == 0
+        return Dict{String,Any}()
+    end
+
+    class_id, _, _, saveobj_id, normobj_id, _ = get_object_metadata(object_id)
+    if saveobj_id != 0
+        saveobj_ret_type = true
+        obj_type_id = saveobj_id
+    else
+        saveobj_ret_type = false
+        obj_type_id = normobj_id
+    end
+
+    prop_map = get_default_properties(class_id)
+    merge!(prop_map, get_saved_properties(obj_type_id, saveobj_ret_type))
+    # TODO: Add dynamic properties
+    return prop_map
+end
+
+function load_mcos_object(metadata::Any, type_name::String)
+    # TODO: Add support for handle class objects
+
+    if type_name != "MCOS"
+        @warn "Loading Type:$type_name is not implemented. Returning metadata."
+        return metadata
+    end
+
+    if isa(metadata, Dict)
+        # TODO: Load Enumeration Instances
+        @warn "Loading enumeration instances are not supported. Returning Metadata"
+        return metadata
+    end
+
+    if !(metadata isa Array{UInt32})
+        @warn "Expected MCOS metadata to be an Array{UInt32}, got $(typeof(metadata)). Returning metadata."
+        return metadata
+    end
+
+    if metadata[1, 1] != 0xDD000000
+        @warn "MCOS object metadata is corrupted. Returning raw data."
+        return metadata
+    end
+
+    ndims = metadata[2, 1]
+    dims = metadata[3:2+ndims, 1]
+    nobjects = prod(dims)
+    object_ids = metadata[3+ndims:2+ndims+nobjects, 1]
+
+    class_id = metadata[end, 1]
+    classname = get_classname(class_id)
+
+    object_arr = Array{Dict{String,Any}}(undef, convert(Vector{Int}, dims)...)
+    for i = 1:length(object_arr)
+        prop_dict = get_properties(object_ids[i])
+        prop_dict["__class__"] = classname
+        object_arr[i] = prop_dict
+    end
+
+    return object_arr
+
+end
+
+end

src/MAT_v5.jl

@@ -29,6 +29,8 @@ module MAT_v5
 using CodecZlib, BufferedStreams, HDF5, SparseArrays
 import Base: read, write, close
 
+using ..MAT_subsys
+
 round_uint8(data) = round.(UInt8, data)
 complex_array(a, b) = complex.(a, b)
 
@@ -246,7 +248,7 @@ function read_sparse(f::IO, swap_bytes::Bool, dimensions::Vector{Int32}, flags::
     end
     if length(ir) > length(pr)
         # Fix for Issue #169, xref https://github.com/JuliaLang/julia/pull/40523
-        #= 
+        #=
         # The following expression must be obeyed according to
         # https://github.com/JuliaLang/julia/blob/b3e4341d43da32f4ab6087230d98d00b89c8c004/stdlib/SparseArrays/src/sparsematrix.jl#L86-L90
         @debug "SparseMatrixCSC" m n jc ir pr
@@ -311,6 +313,18 @@ function read_string(f::IO, swap_bytes::Bool, dimensions::Vector{Int32})
     data
 end
 
+function read_opaque(f::IO, swap_bytes::Bool)
+    type_name = String(read_element(f, swap_bytes, UInt8))
+    classname = String(read_element(f, swap_bytes, UInt8))
+
+    if classname == "FileWrapper__"
+        return read_matrix(f, swap_bytes)
+    end
+
+    _, metadata = read_matrix(f, swap_bytes)
+    return MAT_subsys.load_mcos_object(metadata, type_name)
+end
+
 # Read matrix data
 function read_matrix(f::IO, swap_bytes::Bool)
     (dtype, nbytes) = read_header(f, swap_bytes)
@@ -332,15 +346,10 @@ function read_matrix(f::IO, swap_bytes::Bool)
     flags = read_element(f, swap_bytes, UInt32)
     class = flags[1] & 0xFF
 
-    if class == mxOPAQUE_CLASS
-        s0 = read_data(f, swap_bytes)
-        s1 = read_data(f, swap_bytes)
-        s2 = read_data(f, swap_bytes)
-        arr = read_matrix(f, swap_bytes)
-        return ("__opaque__", Dict("s0"=>s0, "s1"=>s1, "s2"=>s2, "arr"=>arr))
+    if class != mxOPAQUE_CLASS
+        dimensions = read_element(f, swap_bytes, Int32)
     end
 
-    dimensions = read_element(f, swap_bytes, Int32)
     name = String(read_element(f, swap_bytes, UInt8))
 
     local data
@@ -354,6 +363,8 @@ function read_matrix(f::IO, swap_bytes::Bool)
         data = read_string(f, swap_bytes, dimensions)
     elseif class == mxFUNCTION_CLASS
         data = read_matrix(f, swap_bytes)
+    elseif class == mxOPAQUE_CLASS
+        data = read_opaque(f, swap_bytes)
     else
         if (flags[1] & (1 << 9)) != 0 # logical
             data = read_data(f, swap_bytes, Bool, dimensions)
@@ -375,14 +386,41 @@ matopen(ios::IOStream, endian_indicator::UInt16) =
 
 # Read whole MAT file
 function read(matfile::Matlabv5File)
-    seek(matfile.ios, 128)
     vars = Dict{String, Any}()
+
+    seek(matfile.ios, 116)
+    subsys_offset = read_bswap(matfile.ios, matfile.swap_bytes, UInt64)
+    if subsys_offset == 0x2020202020202020
+        subsys_offset = 0
+    end
+    if subsys_offset != 0
+        read_subsystem(matfile.ios, matfile.swap_bytes, subsys_offset)
+    end
+
+    seek(matfile.ios, 128)
     while !eof(matfile.ios)
+        offset = position(matfile.ios)
+        if offset == subsys_offset
+            # Skip reading subsystem again
+            (_, nbytes) = read_header(matfile.ios, matfile.swap_bytes)
+            skip(matfile.ios, nbytes)
+            continue
+        end
         (name, data) = read_matrix(matfile.ios, matfile.swap_bytes)
         vars[name] = data
     end
     vars
 end
+
+function read_subsystem(ios::IOStream, swap_bytes::Bool, offset::UInt64)
+    seek(ios, offset)
+    (_, subsystem_data) = read_matrix(ios, swap_bytes)
+    buf = IOBuffer(vec(subsystem_data))
+    seek(buf, 8) # Skip subsystem header
+    _, subsys_data = read_matrix(buf, swap_bytes)
+    MAT_subsys.load_subsys!(subsys_data, swap_bytes)
+end
+
 # Read only variable names from an HDF5 file
 function getvarnames(matfile::Matlabv5File)
     if !isdefined(matfile, :varnames)