add expectedsize and minoutsize methods (#14)

* add expectedsize and minoutsize methods

* more docs [ci skip]

* add size test

Author: bicycle1885

docs/src/references.md

@@ -31,6 +31,8 @@ TranscodingStreams.CodecIdentity.IdentityStream
 
 ```@docs
 TranscodingStreams.Codec
+TranscodingStreams.expectedsize
+TranscodingStreams.minoutsize
 TranscodingStreams.initialize
 TranscodingStreams.finalize
 TranscodingStreams.startproc

src/codec.jl

@@ -12,23 +12,40 @@ Transcoding protocol
 Transcoding proceeds by calling some functions in a specific way. We call this
 "transcoding protocol" and any codec must implement it as described below.
 
-There are four functions for a codec to implement:
+There are six functions for a codec to implement:
+- `expectedsize`: return the expected size of transcoded data
+- `minoutsize`: return the minimum output size of `process`
 - `initialize`: initialize the codec
 - `finalize`: finalize the codec
 - `startproc`: start processing with the codec
 - `process`: process data with the codec.
 
 These are defined in the `TranscodingStreams` and a new codec type must extend
 these methods if necessary.  Implementing a `process` method is mandatory but
-other three are optional.  `initialize`, `finalize`, and `startproc` have a
-default implementation that does nothing.
+others are optional.  `expectedsize`, `minoutsize`, `initialize`, `finalize`,
+and `startproc` have a default implementation.
 
 Your codec type is denoted by `C` and its object by `codec`.
 
 Errors that occur in these methods are supposed to be unrecoverable and the
 stream will go to the panic state. Only `Base.isopen` and `Base.close` are
 available in that state.
 
+### `expectedsize`
+
+The `expectedsize(codec::C, input::Memory)::Int` method takes `codec` and
+`input`, and returns the expected size of transcoded data. This method will be
+used as a hint to determine the size of a data buffer when `transcode` is
+called. A good hint will reduce the number of buffer resizing and hence result
+in better performance.
+
+### `minoutsize`
+
+The `minoutsize(codec::C, input::Memory)::Int` method takes `codec` and `input`,
+and returns the minimum required size of the output memory when `process` is
+called.  For example, an encoder of base64 will write at least four bytes to the
+output and hence it is reasonable to return 4 with this method.
+
 ### `initialize`
 
 The `initialize(codec::C)::Void` method takes `codec` and returns `nothing`.
@@ -84,10 +101,34 @@ abstract type Codec end
 # Methods
 # -------
 
+"""
+    expectedsize(codec::Codec, input::Memory)::Int
+
+Return the expected size of the transcoded `input` with `codec`.
+
+The default method returns `input.size`.
+"""
+function expectedsize(codec::Codec, input::Memory)::Int
+    return input.size
+end
+
+"""
+    minoutsize(codec::Codec, input::Memory)::Int
+
+Return the minimum output size to be ensured when calling `process`.
+
+The default method returns `max(1, div(input.size, 4))`.
+"""
+function minoutsize(codec::Codec, input::Memory)::Int
+    return max(1, div(input.size, 4))
+end
+
 """
     initialize(codec::Codec)::Void
 
 Initialize `codec`.
+
+The default method does nothing.
 """
 function initialize(codec::Codec)
     return nothing
@@ -97,6 +138,8 @@ end
     finalize(codec::Codec)::Void
 
 Finalize `codec`.
+
+The default method does nothing.
 """
 function finalize(codec::Codec)::Void
     return nothing
@@ -106,6 +149,8 @@ end
     startproc(codec::Codec, state::Symbol, error::Error)::Symbol
 
 Start data processing with `codec` of `state`.
+
+The default method does nothing and returns `:ok`.
 """
 function startproc(codec::Codec, state::Symbol, error::Error)::Symbol
     return :ok
@@ -115,6 +160,8 @@ end
     process(codec::Codec, input::Memory, output::Memory, error::Error)::Tuple{Int,Int,Symbol}
 
 Do data processing with `codec`.
+
+There is no default method.
 """
 function process(codec::Codec, input::Memory, output::Memory, error::Error)::Tuple{Int,Int,Symbol}
     # no default method

src/stream.jl

@@ -330,7 +330,9 @@ julia> String(decompressed)
 ```
 """
 function Base.transcode(codec::Codec, data::Vector{UInt8})
-    buffer2 = Buffer(length(data))
+    # Add `minoutsize` because `transcode` will be called at least two times.
+    buffer2 = Buffer(
+        expectedsize(codec, Memory(data)) + minoutsize(codec, Memory(C_NULL, 0)))
     mark!(buffer2)
     stream = TranscodingStream(codec, DevNull, State(Buffer(data), buffer2))
     write(stream, TOKEN_END)
@@ -440,8 +442,9 @@ function process_to_write(stream::TranscodingStream)
 end
 
 function call_process(codec::Codec, state::State, inbuf::Buffer, outbuf::Buffer)
-    makemargin!(outbuf, clamp(div(length(outbuf), 4), 1, DEFAULT_BUFFER_SIZE * 8))
-    Δin, Δout, state.code = process(codec, buffermem(inbuf), marginmem(outbuf), state.error)
+    input = buffermem(inbuf)
+    makemargin!(outbuf, minoutsize(codec, input))
+    Δin, Δout, state.code = process(codec, input, marginmem(outbuf), state.error)
     inbuf.bufferpos += Δin
     outbuf.marginpos += Δout
     outbuf.total += Δout

test/runtests.jl

@@ -203,10 +203,11 @@ end
 
 struct QuadrupleCodec <: TranscodingStreams.Codec end
 
+const Memory = TranscodingStreams.Memory
 function TranscodingStreams.process(
         codec  :: QuadrupleCodec,
-        input  :: TranscodingStreams.Memory,
-        output :: TranscodingStreams.Memory,
+        input  :: Memory,
+        output :: Memory,
         error  :: TranscodingStreams.Error)
     i = j = 0
     while i + 1 ≤ endof(input) && j + 4 ≤ endof(output)
@@ -218,10 +219,16 @@ function TranscodingStreams.process(
     return i, j, input.size == 0 ? (:end) : (:ok)
 end
 
+TranscodingStreams.expectedsize(::QuadrupleCodec, input::Memory) = input.size * 4
+TranscodingStreams.minoutsize(::QuadrupleCodec, ::Memory) = 4
+
 @testset "QuadrupleCodec" begin
     @test transcode(QuadrupleCodec(), b"") == b""
     @test transcode(QuadrupleCodec(), b"a") == b"aaaa"
     @test transcode(QuadrupleCodec(), b"ab") == b"aaaabbbb"
+    data = "x"^1024
+    transcode(QuadrupleCodec(), data)
+    @test (@allocated transcode(QuadrupleCodec(), data)) < sizeof(data) * 5
 end
 
 for pkg in ["CodecZlib", "CodecBzip2", "CodecXz", "CodecZstd", "CodecBase"]