Do not reserve exponentially growing margin (#130)

In fde7eea, buffers were changed to growing exponentially by calling
`makemargin!(buf, X)` with an exponentially growing X. However, this is not the
right way of solving the problem. Indeed, only a single byte of margin is
necessary when removing already-used data from the buffer. Requiring
exponentially increasing margins forces unneeded buffer resizes.

Instead, `makemargin!` itself should exponentially increase buffersize when it
finds that it can't make enough margin, and the callers of `makemargin!` should
request only 1 byte of space, as before.

Author: jakobnissen

Project.toml

@@ -2,7 +2,7 @@ name = "TranscodingStreams"
 uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
 license = "MIT"
 authors = ["Kenta Sato <[email protected]>"]
-version = "0.9.10"
+version = "0.9.11"
 
 [deps]
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

src/buffer.jl

@@ -130,14 +130,19 @@ function makemargin!(buf::Buffer, minsize::Integer; eager::Bool = false)
         buf.bufferpos = buf.marginpos = 1
     end
     if marginsize(buf) < minsize || eager
-        # shift data to left
+        # datapos refer to the leftmost position of data that must not be
+        # discarded. We can left-shift to discard all data before this
         if buf.markpos == 0
+            # If data is not marked we must not discard buffered (nonconsumed) data
             datapos = buf.bufferpos
             datasize = buffersize(buf)
         else
+            # Else, we must not consume marked data
+            # (Since markpos ≤ bufferpos, we do not consume buffered data either) 
             datapos = buf.markpos
             datasize = buf.marginpos - buf.markpos
         end
+        # Shift data left in buffer to make space for new data
         copyto!(buf.data, 1, buf.data, datapos, datasize)
         shift = datapos - 1
         if buf.markpos > 0
@@ -146,9 +151,11 @@ function makemargin!(buf::Buffer, minsize::Integer; eager::Bool = false)
         buf.bufferpos -= shift
         buf.marginpos -= shift
     end
+    # If there is still not enough margin, we expand buffer.
+    # At least enough for minsize, but otherwise 1.5 times
     if marginsize(buf) < minsize
-        # expand data buffer
-        resize!(buf.data, buf.marginpos + minsize - 1)
+        datasize = length(buf.data)
+        resize!(buf.data, max(buf.marginpos + minsize - 1, datasize + div(datasize, 2)))
     end
     @assert marginsize(buf) ≥ minsize
     return marginsize(buf)

src/noop.jl

@@ -167,7 +167,7 @@ function fillbuffer(stream::NoopStream; eager::Bool = false)
     end
     nfilled::Int = 0
     while ((!eager && buffersize(buffer) == 0) || (eager && makemargin!(buffer, 0, eager = true) > 0)) && !eof(stream.stream)
-        makemargin!(buffer, max(1, div(length(buffer), 2)))
+        makemargin!(buffer, 1)
         nfilled += readdata!(stream.stream, buffer)
     end
     buffer.transcoded += nfilled

src/stream.jl

@@ -591,7 +591,7 @@ function fillbuffer(stream::TranscodingStream; eager::Bool = false)
             end
             callstartproc(stream, :read)
         end
-        makemargin!(buffer2, max(1, div(length(buffer2), 2)))
+        makemargin!(buffer2, 1)
         readdata!(stream.stream, buffer2)
         _, Δout = callprocess(stream, buffer2, buffer1)
         nfilled += Δout

test/runtests.jl

@@ -58,9 +58,10 @@ using TranscodingStreams:
     writebyte!(buf, 0x34)
     @test makemargin!(buf, 0) === 15
     writebyte!(buf, 0x99)
-    @test makemargin!(buf, 20) === 20
+    margin_size = makemargin!(buf, 20) 
+    @test margin_size >= 20
     emptybuffer!(buf)
-    @test makemargin!(buf, 0) === 22
+    @test makemargin!(buf, 0) === margin_size + 2
 end
 
 @testset "Memory" begin