initial WIP draft to demonstrate general idea

Author: ssfrr

README.md

@@ -37,6 +37,40 @@ s = query(io)   # io is a stream
 will return a `File` or `Stream` object that also encodes the detected
 file format.
 
+Sometimes you want to read or write files that are larger than your available
+memory, or might be an unknown or infinite length (e.g. reading an audio or
+video stream from a socket). In these cases it might not make sense to process
+the whole file at once, but instead process it a chunk at a time. For these situations FileIO provides the `loadstreaming` and `savestreaming` functions, which return an object that you can `read` or `write`, rather than the file data itself.
+
+This would look something like:
+
+```jl
+using FileIO
+audio = loadstreaming("bigfile.wav")
+try
+    while !eof(audio)
+        chunk = read(audio, 4096) # read 4096 frames
+        # process the chunk
+    end
+finally
+    close(stream)
+end
+```
+
+or use `do` syntax to auto-close the stream:
+
+```jl
+using FileIO
+do loadstreaming("bigfile.wav") audio
+    while !eof(audio)
+        chunk = read(audio, 4096) # read 4096 frames
+        # process the chunk
+    end
+end
+```
+
+Note that in these cases you may want to use `read!` with a pre-allocated buffer for maximum efficiency.
+
 ## Adding new formats
 
 You register a new format by adding `add_format(fmt, magic,
@@ -139,6 +173,46 @@ automatically even if the code inside the `do` scope throws an error.)
 Conversely, `load(::Stream)` and `save(::Stream)` should not close the
 input stream.
 
+`loadstreaming` and `savestreaming` use the same query mechanism, but return a decoded stream that users can `read` or `write`. You should also implement a `close` method on your reader or writer type. Just like with `load` and `save`, if the user provided a filename, your `close` method should be responsible for closing any streams you opened in order to read or write the file. If you are given a `Stream`, your `close` method should only do the clean up for your reader or writer type, not close the stream.
+
+```julia
+struct WAVReader
+    io::IO
+    ownstream::Bool
+end
+
+function read(reader::WAVReader, frames::Int)
+    # read and decode audio samples from reader.io
+end
+
+function close(reader::WAVReader)
+    # do whatever cleanup the reader needs
+    if reader.ownstream
+        close(reader.io)
+    end
+end
+loadstreaming(f::File{format"WAV"}) = WAVReader(open(f), ownstream=true)
+loadstreaming(s::Stream{format"WAV"}) = WAVReader(s, ownstream=false)
+# FileIO has fallback functions that make these work using `do` syntax as well.
+```
+
+If you choose to implement `loadstreaming` and `savestreaming` in your package,
+you can easily add `save` and `load` methods in the form of:
+
+```julia
+function save(q::Formatted{format"WAV"}, data, args...; kwargs...)
+    savestreaming(args...; kwargs...) do stream
+        write(stream, data)
+    end
+end
+
+function load(q::Formatted{format"WAV"}, args...; kwargs...)
+    savestreaming(args...; kwargs...) do stream
+        readall(stream)
+    end
+end
+```
+
 ## Help
 
 You can get an API overview by typing `?FileIO` at the REPL prompt.

src/FileIO.jl

@@ -17,9 +17,11 @@ export DataFormat,
        file_extension,
        info,
        load,
+       loadstreaming,
        magic,
        query,
        save,
+       savestreaming,
        skipmagic,
        stream,
        unknown
@@ -40,7 +42,9 @@ include("registry.jl")
 
 - `load([filename|stream])`: read data in formatted file, inferring the format
 - `load(File(format"PNG",filename))`: specify the format manually
+- `loadstreaming(f)`: similar to `load`, except that it returns an object that can be read from
 - `save(filename, data...)` for similar operations involving saving data
+- `savestreaming(f)`: similar to `save`, except that it returns an object that can be written to
 
 - `io = open(f::File, args...)` opens a file
 - `io = stream(s::Stream)` returns the IOStream from the query object `s`

src/loadsave.jl

@@ -49,17 +49,50 @@ the magic bytes are essential.
 - `load(File(format"PNG",filename))` specifies the format directly, and bypasses inference.
 - `load(f; options...)` passes keyword arguments on to the loader.
 """
-load(s::Union{AbstractString,IO}, args...; options...) =
-    load(query(s), args...; options...)
+load
+
+"""
+Some packages may implement a streaming API, where the contents of the file can
+be read in chunks and processed, rather than all at once. Reading from these
+higher-level streams should return a formatted object, like an image or chunk of
+video or audio.
+
+- `loadstreaming(filename)` loads the contents of a formatted file, trying to infer
+the format from `filename` and/or magic bytes in the file. It returns a streaming
+type that can be read from in chunks, rather than loading the whole contents all
+at once
+- `loadstreaming(strm)` loads the stream from an `IOStream` or similar object. In this case,
+the magic bytes are essential.
+- `load(File(format"PNG",filename))` specifies the format directly, and bypasses inference.
+- `load(f; options...)` passes keyword arguments on to the loader.
+"""
+loadstreaming
 
 """
 - `save(filename, data...)` saves the contents of a formatted file,
 trying to infer the format from `filename`.
 - `save(Stream(format"PNG",io), data...)` specifies the format directly, and bypasses inference.
 - `save(f, data...; options...)` passes keyword arguments on to the saver.
 """
-save(s::Union{AbstractString,IO}, data...; options...) =
-    save(query(s), data...; options...)
+save
+
+"""
+Some packages may implement a streaming API, where the contents of the file can
+be written in chunks, rather than all at once. These higher-level streams should
+accept formatted objects, like an image or chunk of video or audio.
+
+- `savestreaming(filename, data...)` saves the contents of a formatted file,
+trying to infer the format from `filename`.
+- `savestreaming(Stream(format"PNG",io), data...)` specifies the format directly, and bypasses inference.
+- `savestreaming(f, data...; options...)` passes keyword arguments on to the saver.
+"""
+savestreaming
+
+
+for fn in (:load, :loadstreaming, :save, :savestreaming)
+    @eval $fn(s::Union{AbstractString,IO}, data...; options...) =
+        $fn(query(s), data...; options...)
+end
 
 function save(s::Union{AbstractString,IO}; options...)
     data -> save(s, data; options...)
@@ -73,51 +106,79 @@ function save{sym}(df::Type{DataFormat{sym}}, f::AbstractString, data...; option
                        $data...; $options...)))
 end
 
+function savestreaming{sym}(df::Type{DataFormat{sym}}, s::IO, data...; options...)
+    libraries = applicable_savers(df)
+    checked_import(libraries[1])
+    eval(Main, :($savestreaming($Stream($(DataFormat{sym}), $s),
+                                $data...; $options...)))
+
 function save{sym}(df::Type{DataFormat{sym}}, s::IO, data...; options...)
     libraries = applicable_savers(df)
     checked_import(libraries[1])
     eval(Main, :($save($Stream($(DataFormat{sym}), $s),
                        $data...; $options...)))
+
+function savestreaming{sym}(df::Type{DataFormat{sym}}, f::AbstractString, data...; options...)
+    libraries = applicable_savers(df)
+    checked_import(libraries[1])
+    eval(Main, :($savestreaming($File($(DataFormat{sym}), $f),
+                                $data...; $options...)))
+end
+
+# do-syntax for streaming IO
+for fn in (:loadstreaming, :savestreaming)
+    @eval function $fn(f::Function, args...; kwargs...)
+        str = $fn(args...; kwargs...)
+        try
+            f(str)
+        finally
+            close(str)
+        end
+    end
 end
 
 
 # Fallbacks
-function load{F}(q::Formatted{F}, args...; options...)
-    if unknown(q)
-        isfile(filename(q)) || open(filename(q))  # force systemerror
-        throw(UnknownFormat(q))
-    end
-    libraries = applicable_loaders(q)
-    failures  = Any[]
-    for library in libraries
-        try
-            Library = checked_import(library)
-            if !has_method_from(methods(Library.load), Library)
-                throw(LoaderError(string(library), "load not defined"))
+for fn in (:load, :loadstreaming)
+    @eval function $fn{F}(q::Formatted{F}, args...; options...)
+        if unknown(q)
+            isfile(filename(q)) || open(filename(q))  # force systemerror
+            throw(UnknownFormat(q))
+        end
+        libraries = applicable_loaders(q)
+        failures  = Any[]
+        for library in libraries
+            try
+                Library = checked_import(library)
+                if !has_method_from(methods(Library.$fn), Library)
+                    throw(LoaderError(string(library), "$fn not defined"))
+                end
+                return eval(Main, :($(Library.$fn)($q, $args...; $options...)))
+            catch e
+                push!(failures, (e, q))
             end
-            return eval(Main, :($(Library.load)($q, $args...; $options...)))
-        catch e
-            push!(failures, (e, q))
         end
+        handle_exceptions(failures, "loading \"$(filename(q))\"")
     end
-    handle_exceptions(failures, "loading \"$(filename(q))\"")
 end
-function save{F}(q::Formatted{F}, data...; options...)
-    unknown(q) && throw(UnknownFormat(q))
-    libraries = applicable_savers(q)
-    failures  = Any[]
-    for library in libraries
-        try
-            Library = checked_import(library)
-            if !has_method_from(methods(Library.save), Library)
-                throw(WriterError(string(library), "save not defined"))
+for fn in (:save, :savestreaming)
+    @eval function $fn{F}(q::Formatted{F}, data...; options...)
+        unknown(q) && throw(UnknownFormat(q))
+        libraries = applicable_savers(q)
+        failures  = Any[]
+        for library in libraries
+            try
+                Library = checked_import(library)
+                if !has_method_from(methods(Library.$fn), Library)
+                    throw(WriterError(string(library), "$fn not defined"))
+                end
+                return eval(Main, :($(Library.$fn)($q, $data...; $options...)))
+            catch e
+                push!(failures, (e, q))
             end
-            return eval(Main, :($(Library.save)($q, $data...; $options...)))
-        catch e
-            push!(failures, (e, q))
         end
+        handle_exceptions(failures, "saving \"$(filename(q))\"")
     end
-    handle_exceptions(failures, "saving \"$(filename(q))\"")
 end
 
 function has_method_from(mt, Library)