diff --git a/.gitignore b/.gitignore index b25c15b..93bfd12 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ *~ +.DS_Store diff --git a/src/MAT_HDF5.jl b/src/MAT_HDF5.jl index 22f1efa..9d2e80c 100644 --- a/src/MAT_HDF5.jl +++ b/src/MAT_HDF5.jl @@ -172,6 +172,11 @@ function m_read(dset::HDF5Dataset) # Regular arrays of values # Convert to Julia type + if !haskey(str2type_matlab, mattype) + @warn "Class not convertable - skipping field - set value to missing" + return missing + end + T = str2type_matlab[mattype] # Check for a COMPOUND data set, and if so handle complex numbers specially @@ -207,6 +212,11 @@ function m_read(g::HDF5Group) # This matrix is not empty. ir = add!(convert(Vector{Int}, read(g, "ir")), 1) dset = g["data"] + if !haskey(str2type_matlab, mattype) + @warn "Class not convertable - skipping field - set value to missing" + return missing + end + T = str2type_matlab[mattype] try dtype = datatype(dset) @@ -258,6 +268,9 @@ function read(f::MatlabHDF5File, name::String) obj = f.plain[name] try val = m_read(obj) + catch + @warn "Field not readable - skipping field - set value to missing" + val = missing finally close(obj) end diff --git a/src/MAT_v5.jl b/src/MAT_v5.jl index 789affb..a6635d6 100644 --- a/src/MAT_v5.jl +++ b/src/MAT_v5.jl @@ -100,8 +100,13 @@ function read_bswap(f::IO, swap_bytes::Bool, d::AbstractArray{T}) where T d end -skip_padding(f::IO, nbytes::Int, hbytes::Int) = if nbytes % hbytes != 0 - skip(f, hbytes-(nbytes % hbytes)) +function skip_padding(f::IO, nbytes::Int, hbytes::Int) + if nbytes % hbytes != 0 + skip(f, hbytes-(nbytes % hbytes)) + return hbytes-(nbytes % hbytes) + else + return 0 + end end # Read data type and number of bytes at the start of a data element @@ -121,8 +126,9 @@ end function read_element(f::IO, swap_bytes::Bool, ::Type{T}) where T (dtype, nbytes, hbytes) = read_header(f, swap_bytes) data = read_bswap(f, swap_bytes, T, Int(div(nbytes, sizeof(T)))) - skip_padding(f, nbytes, hbytes) - data + skipped_bytes = skip_padding(f, nbytes, hbytes) + bytes_read = nbytes + hbytes + skipped_bytes + return data, bytes_read end # Read data element as encoded type @@ -168,10 +174,11 @@ end function read_struct(f::IO, swap_bytes::Bool, dimensions::Vector{Int32}, is_object::Bool) if is_object - class = String(read_element(f, swap_bytes, UInt8)) + class = String(read_element(f, swap_bytes, UInt8)[1]) end - field_length = read_element(f, swap_bytes, Int32)[1] - field_names = read_element(f, swap_bytes, UInt8) + field_lengths, _ = read_element(f, swap_bytes, Int32) + field_length = field_lengths[1] + field_names, _ = read_element(f, swap_bytes, UInt8) n_fields = div(length(field_names), field_length) # Get field names as strings @@ -232,11 +239,11 @@ function read_sparse(f::IO, swap_bytes::Bool, dimensions::Vector{Int32}, flags:: m = isempty(dimensions) ? 0 : dimensions[1] n = length(dimensions) <= 1 ? 0 : dimensions[2] - ir = plusone!(convert(Vector{Int}, read_element(f, swap_bytes, Int32))) - jc = plusone!(convert(Vector{Int}, read_element(f, swap_bytes, Int32))) + ir = plusone!(convert(Vector{Int}, read_element(f, swap_bytes, Int32)[1])) + jc = plusone!(convert(Vector{Int}, read_element(f, swap_bytes, Int32)[1])) if (flags[1] & (1 << 9)) != 0 # logical # WTF. For some reason logical sparse matrices are tagged as doubles. - pr = read_element(f, swap_bytes, Bool) + pr = read_element(f, swap_bytes, Bool)[1] else pr = read_data(f, swap_bytes) if (flags[1] & (1 << 11)) != 0 # complex @@ -316,9 +323,10 @@ function read_matrix(f::IO, swap_bytes::Bool) return ("", Matrix{Union{}}(undef, 0, 0)) end - flags = read_element(f, swap_bytes, UInt32) - dimensions = read_element(f, swap_bytes, Int32) - name = String(read_element(f, swap_bytes, UInt8)) + flags, f_bytes = read_element(f, swap_bytes, UInt32) + dimensions, d_bytes = read_element(f, swap_bytes, Int32) + raw_name, n_bytes = read_element(f, swap_bytes, UInt8) + name = String(raw_name) class = flags[1] & 0xFF local data @@ -330,6 +338,11 @@ function read_matrix(f::IO, swap_bytes::Bool) data = read_sparse(f, swap_bytes, dimensions, flags) elseif class == mxCHAR_CLASS && length(dimensions) <= 2 data = read_string(f, swap_bytes, dimensions) + elseif class > length(CONVERT_TYPES) # undocumented classes + @warn "Class not convertable - skipping field - set value to missing" + remaining_bytes = nbytes - f_bytes - d_bytes - n_bytes + skip(f, remaining_bytes) + data = missing else if (flags[1] & (1 << 9)) != 0 # logical data = read_data(f, swap_bytes, Bool, dimensions) @@ -378,7 +391,7 @@ function getvarnames(matfile::Matlabv5File) read_element(f, matfile.swap_bytes, UInt32) read_element(f, matfile.swap_bytes, Int32) - varnames[String(read_element(f, matfile.swap_bytes, UInt8))] = offset + varnames[String(read_element(f, matfile.swap_bytes, UInt8)[1])] = offset seek(matfile.ios, offset+nbytes+hbytes) end diff --git a/test/read.jl b/test/read.jl index 6e00483..9567014 100644 --- a/test/read.jl +++ b/test/read.jl @@ -1,8 +1,9 @@ using MAT, Test -function check(filename, result) +function check(filename, result; skip_keys=String[]) matfile = matopen(filename) for (k, v) in result + k in skip_keys && continue @test exists(matfile, k) got = read(matfile, k) if !isequal(got, v) || (typeof(got) != typeof(v) && (!isa(got, String) || !(isa(v, String)))) @@ -20,24 +21,28 @@ function check(filename, result) """) end end - @test union!(Set(), names(matfile)) == union!(Set(), keys(result)) + if isempty(skip_keys) + @test union!(Set(), names(matfile)) == union!(Set(), keys(result)) + end close(matfile) - mat = matread(filename) - if !isequal(mat, result) - error(""" - Data mismatch reading $filename ($format) + if isempty(skip_keys) + mat = matread(filename) + if !isequal(mat, result) + error(""" + Data mismatch reading $filename ($format) - Got: + Got: - $(repr(mat)) + $(repr(mat)) - Expected: + Expected: - $(repr(result)) - """) - close(matfile) - return false + $(repr(result)) + """) + close(matfile) + return false + end end return true @@ -136,6 +141,18 @@ for _format in ["v6", "v7", "v7.3"] ) check("sparse.mat", result) + result = Dict( + "t" => Dict( + "int" => 66.0, + "struct" => Dict{String,Any}( + "b"=>[5.0 6.0 7.0], + "a"=>[1.0 2.0 3.0]), + "skip_field" => missing + ) + ) + + check("skip_field.mat", result, skip_keys=["", "#subsystem#"]) + matfile = matopen("partial.mat") var1 = read(matfile, "var1") @assert var1[28, 33] == 5 diff --git a/test/v6/skip_field.mat b/test/v6/skip_field.mat new file mode 100644 index 0000000..d44dce2 Binary files /dev/null and b/test/v6/skip_field.mat differ diff --git a/test/v7.3/skip_field.mat b/test/v7.3/skip_field.mat new file mode 100644 index 0000000..e67d60d Binary files /dev/null and b/test/v7.3/skip_field.mat differ diff --git a/test/v7/skip_field.mat b/test/v7/skip_field.mat new file mode 100644 index 0000000..163bf34 Binary files /dev/null and b/test/v7/skip_field.mat differ diff --git a/test/write.jl b/test/write.jl index 3b8c910..b6b219d 100644 --- a/test/write.jl +++ b/test/write.jl @@ -14,7 +14,17 @@ function test_write(data) end if !isequal(result, data) - error("Data mismatch") + error(""" + Data mismatch + + Got: + + $(repr(result)) + + Expected: + + $(repr(data)) + """) end end