Merge pull request #31 from JuliaIO/sd/multimagic

first take on multiple magic bytes

Author: SimonDanisch

src/query.jl

@@ -29,6 +29,7 @@ const magic_func = Array(Pair, 0)    # for formats with complex magic #s
 `add_format(fmt, magic, extention)` registers a new `DataFormat`.
 For example:
 
+    add_format(format"PNG", (UInt8[0x4d,0x4d,0x00,0x2b], UInt8[0x49,0x49,0x2a,0x00]), [".tiff", ".tif"])
     add_format(format"PNG", [0x89,0x50,0x4e,0x47,0x0d,0x0a,0x1a,0x0a], ".png")
     add_format(format"NRRD", "NRRD", [".nrrd",".nhdr"])
 
@@ -47,6 +48,22 @@ function add_format{sym}(fmt::Type{DataFormat{sym}}, magic::Union(Tuple,Abstract
     fmt
 end
 
+# for multiple magic bytes
+function add_format{sym, T <: Vector{Uint8}, N}(fmt::Type{DataFormat{sym}}, magics::NTuple{N, T}, extension)
+    haskey(sym2info, sym) && error("format ", fmt, " is already registered")
+    magics = map(canonicalize_magic, magics)
+    for magic in magics
+        rng = searchsorted(magic_list, magic, lt=magic_cmp)
+        if !isempty(magic) && !isempty(rng)
+            error("magic bytes ", magic, " are already registered")
+        end
+        insert!(magic_list, first(rng), Pair(magic, sym))  # m=>sym in 0.4
+    end
+    sym2info[sym] = (magics, extension)
+    add_extension(extension, sym)
+    fmt
+end
+
 # For when "magic" is supplied as a function (see the HDF5 example in
 # registry.jl)
 function add_format{sym}(fmt::Type{DataFormat{sym}}, magic, extension)
@@ -62,6 +79,18 @@ end
 """ ->
 function del_format{sym}(fmt::Type{DataFormat{sym}})
     magic, extension = sym2info[sym]
+    del_magic(magic, sym)
+    delete!(sym2info, sym)
+    del_extension(extension)
+    nothing
+end
+
+# Deletes mutliple magic bytes
+del_magic(magic::Tuple, sym) = for m in magic
+    del_magic(m, sym)
+end
+# Deletes single magic bytes
+function del_magic{N}(magic::NTuple{N, Uint8}, sym)
     rng = searchsorted(magic_list, magic, lt=magic_cmp)
     if length(magic) == 0
         fullrng = rng
@@ -77,8 +106,6 @@ function del_format{sym}(fmt::Type{DataFormat{sym}})
     end
     @assert length(rng) == 1
     deleteat!(magic_list, first(rng))
-    delete!(sym2info, sym)
-    del_extension(extension)
     nothing
 end
 
@@ -242,11 +269,37 @@ For a plain IO object, you can use `skipmagic(io, fmt)`.
 skipmagic{F}(s::Stream{F}) = (skipmagic(stream(s), F); s)
 function skipmagic{sym}(io, fmt::Type{DataFormat{sym}})
     magic, _ = sym2info[sym]
-    if !isa(magic, Function)
-        seek(io, length(magic))
+    skipmagic(io, magic)
+    nothing
+end
+skipmagic(io, magic::Function) = nothing
+skipmagic{N}(io, magic::NTuple{N,UInt8}) = seek(io, length(magic))
+function skipmagic(io, magic::Tuple)
+    lengths = map(length, magic)
+    all(x->lengths[1] == x, lengths) && return seek(io, lengths[1]) # it doesn't matter what magic bytes get skipped as they all have the same length
+    magic = [magic...]
+    sort!(magic, lt=(a,b)-> length(a)>= length(b)) # start with longest first, to avoid overlapping magic bytes
+    seekend(io)
+    len = position(io)
+    seekstart(io)
+    filter!(x-> length(x) <= len, magic) # throw out magic bytes that are longer than IO
+    tmp = readbytes(io, length(first(magic))) # now, first is both the longest and guaranteed to fit into io, so we can just read the bytes
+    for m in magic
+        if magic_equal(m, tmp)
+            seek(io, length(m))
+            return nothing
+        end
     end
+    error("tried to skip magic bytes of an IO that does not contain the magic bytes of the format. IO: $io")
+end
+function magic_equal(magic, buffer)
+    for (i,elem) in enumerate(magic)
+        buffer[i] != elem && return false
+    end
+    true
 end
 
+
 unknown{F}(::File{F}) = unknown(F)
 unknown{F}(::Stream{F}) = unknown(F)
 
@@ -257,13 +310,13 @@ function query(filename::AbstractString)
     _, ext = splitext(filename)
     if haskey(ext2sym, ext)
         sym = ext2sym[ext]
-        len = lenmagic(sym)
-        if length(len) == 1 && (all(x->x==0, len) || !isfile(filename)) # we only found one candidate and there is no magic bytes, or no file, trust the extension
+        no_magic = !hasmagic(sym)
+        if lensym(sym) == 1 && (no_magic || !isfile(filename)) # we only found one candidate and there is no magic bytes, or no file, trust the extension
             return File{DataFormat{sym}}(filename)
-        elseif !isfile(filename) && length(len) > 1
+        elseif !isfile(filename) && lensym(sym) > 1
             error("no file for check of magic bytes and multiple extensions possible: $sym")
         end
-        if any(x->x==0, len)
+        if no_magic && !hasfunction(sym)
             error("Some formats with extension ", ext, " have no magic bytes; use `File{format\"FMT\"}(filename)` to resolve the ambiguity.")
         end
     end
@@ -272,11 +325,19 @@ function query(filename::AbstractString)
     file!(query(open(filename), filename))
 end
 
-lenmagic(s::Symbol) = lenm(sym2info[s][1])
-lenmagic(v::Vector) = map(lenmagic, v)
+lensym(s::Symbol) = 1
+lensym(v::Vector) = length(v)
+
+hasmagic(s::Symbol) = hasmagic(sym2info[s][1])
+hasmagic(v::Vector) = any(hasmagic, v)
+
+hasmagic(t::Tuple) = !isempty(t)
+hasmagic(::Any) = false   # for when magic is a function
 
-lenm(t::Tuple) = length(t)
-lenm(::Any) = -1   # for when magic is a function
+hasfunction(s::Symbol) = hasfunction(sym2info[s][1])
+hasfunction(v::Vector) = any(hasfunction, v)
+hasfunction(s::Any) = true #has function
+hasfunction(s::Tuple) = false #has magic
 
 @doc """
 `query(io, [filename])` returns a `Stream` object with information about the

src/registry.jl

@@ -78,7 +78,8 @@ add_saver(format"PSD", :ImageMagick)
 add_format(format"RGB", UInt8[0x01,0xda,0x01,0x01,0x00,0x03], ".rgb")
 add_loader(format"RGB", :ImageMagick)
 add_saver(format"RGB", :ImageMagick)
-add_format(format"TIFF", UInt8[0x4d,0x4d,0x00,0x2b], ".tiff")
+
+add_format(format"TIFF", (UInt8[0x4d,0x4d,0x00,0x2a], UInt8[0x4d,0x4d,0x00,0x2b], UInt8[0x49,0x49,0x2a,0x00]), [".tiff", "tif"])
 add_loader(format"TIFF", :ImageMagick)
 add_saver(format"TIFF", :ImageMagick)
 add_format(format"WMF", UInt8[0xd7,0xcd,0xc6,0x9a], ".wmf")
@@ -147,7 +148,7 @@ add_saver(format"HDF5", :HDF5)
 
 function detect_stlascii(io)
     try
-        position(io) != 0 && return false
+        position(io) != 0 && (seekstart(io); return false)
         seekend(io)
         len = position(io)
         seekstart(io)
@@ -163,7 +164,7 @@ function detect_stlbinary(io)
     const size_header = 80+sizeof(Uint32)
     const size_triangleblock = (4*3*sizeof(Float32)) + sizeof(Uint16)
 
-    position(io) != 0 && return false
+    position(io) != 0 && (seekstart(io); return false)
     seekend(io)
     len = position(io)
     seekstart(io)
@@ -172,12 +173,13 @@ function detect_stlbinary(io)
     skip(io, 80) # skip header
     number_of_triangle_blocks = read(io, Uint32)
      #1 normal, 3 vertices in Float32 + attrib count, usually 0
-    len != (number_of_triangle_blocks*size_triangleblock)+size_header && return false
+    len != (number_of_triangle_blocks*size_triangleblock)+size_header && (seekstart(io); return false)
     skip(io, number_of_triangle_blocks*size_triangleblock-sizeof(Uint16))
     attrib_byte_count = read(io, Uint16) # read last attrib_byte
-    attrib_byte_count != zero(Uint16) && return false # should be zero as not used
-    eof(io) && return true
-    false
+    attrib_byte_count != zero(Uint16) && (seekstart(io); return false) # should be zero as not used
+    result = eof(io) # if end of file, we have a stl!
+    seekstart(io)
+    return result
 end
 add_format(format"STL_ASCII", detect_stlascii, [".stl", ".STL"])
 add_format(format"STL_BINARY", detect_stlbinary, [".stl", ".STL"])

test/files/magic1.tiff

@@ -0,0 +1,25 @@
+
+Michael J. Lyons, Shigeru Akemastu, Miyuki Kamachi, Jiro Gyoba.
+Coding Facial Expressions with Gabor Wavelets, 3rd IEEE International Conference on Automatic Face and Gesture Recognition, pp. 200-205 (1998).
+
+Information about the JAFFE database is available at:
+
+http://www.kasrl.org/jaffe.html
+
+Images are in .tiff format, no compression.
+
+Included below, are files containing semantic rating data from
+psychological experiments using the images. I send it because I
+believe it is important to realize that expression are never pure
+expressions of one emotion, but always admixtures of different
+emotions. The expression labels on the images just represent the
+predominant expression in that image - the expression that the subject
+was asked to pose.
+
+Best regards,
+
+Michael J. Lyons, Ph.D.
+Professor of Image Arts and Sciences
+Ritsumeikan University
+Kyoto, Japan
+michael (dot) lyons(you know)gmail (dot) com

test/files/magic2.tiff

@@ -140,6 +140,44 @@ try
         q = query( "some_non_existant_file.mmm")
         @fact typeof(q) --> File{format"MAGIC"}
 
+        add_format(format"DOUBLE_MAGIC", (UInt8[0x4d,0x4d,0x00,0x2a], UInt8[0x4d,0x4d,0x00]), ".dd2")
+
+        fn = string(tempname(), ".dd2")
+        open(fn, "w") do file
+            write(file, UInt8[0x4d,0x4d,0x00,0x2a])
+            write(file, randstring(19))
+        end
+        q = query(fn)
+        @fact typeof(q) --> File{format"DOUBLE_MAGIC"}
+        io = open(q)
+        skipmagic(io)
+        @fact position(io) --> 4
+        close(io)
+        rm(fn)
+
+        open(fn, "w") do file
+            write(file, UInt8[0x4d,0x4d,0x00])
+            write(file, randstring(19))
+        end
+        q = query(fn)
+        @fact typeof(q) --> File{format"DOUBLE_MAGIC"}
+        io = open(q)
+        @fact file_extension(q) --> ".dd2"
+        skipmagic(io)
+        @fact position(io) --> 3
+        close(io)
+        open(fn, "w") do file
+            write(file, randstring(19)) # corrupt magic bytes
+        end
+        open(fn, "r") do file
+            @fact_throws skipmagic(file)
+        end
+        rm(fn)
+        lene0 = length(FileIO.ext2sym)
+        lenm0 = length(FileIO.magic_list)
+        del_format(format"DOUBLE_MAGIC")
+        @fact lene0 - 1 --> length(FileIO.ext2sym)
+        @fact lenm0 - 2 --> length(FileIO.magic_list)
     end
 
     del_format(format"JUNK")  # This triggers del_extension for multiple extensions
@@ -161,10 +199,27 @@ facts("STL detection") do
     @fact typeof(q) --> File{format"STL_ASCII"}
     q = query(joinpath(file_dir, "binary_stl_from_solidworks.STL"))
     @fact typeof(q) --> File{format"STL_BINARY"}
+    open(q) do io 
+        @fact position(io) --> 0
+        skipmagic(io)
+        @fact position(io) --> 0 # no skipping for functions
+    end
 end
 facts("PLY detection") do 
     q = query(joinpath(file_dir, "ascii.ply"))
     @fact typeof(q) --> File{format"PLY_ASCII"}
     q = query(joinpath(file_dir, "binary.ply"))
     @fact typeof(q) --> File{format"PLY_BINARY"}
+
 end
+facts("Multiple Magic bytes") do 
+    q = query(joinpath(file_dir, "magic1.tiff"))
+    @fact typeof(q) --> File{format"TIFF"}
+    q = query(joinpath(file_dir, "magic2.tiff"))
+    @fact typeof(q) --> File{format"TIFF"}
+    open(q) do io
+        @fact position(io) --> 0
+        skipmagic(io)
+        @fact position(io) --> 4
+    end
+end