Skip to content

Commit fc1bd81

Browse files
author
Chris Busbey
committed
parser shifts buffer to reduce large block allocations
1 parent 7c3f271 commit fc1bd81

File tree

2 files changed

+96
-105
lines changed

2 files changed

+96
-105
lines changed

‎parser.go‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,10 @@ const (
1515
var bufferPool internal.BufferPool
1616

1717
type parser struct {
18-
buffer []byte
19-
reader io.Reader
20-
lastRead time.Time
18+
//buffer is a slice of bigBuffer
19+
bigBuffer, buffer []byte
20+
reader io.Reader
21+
lastRead time.Time
2122
}
2223

2324
func newParser(reader io.Reader) *parser {
@@ -26,7 +27,23 @@ func newParser(reader io.Reader) *parser {
2627

2728
func (p *parser) readMore() (int, error) {
2829
if len(p.buffer) == cap(p.buffer) {
29-
newBuffer := make([]byte, len(p.buffer), len(p.buffer)+defaultBufSize)
30+
var newBuffer []byte
31+
switch {
32+
//initialize the parser
33+
case len(p.bigBuffer) == 0:
34+
p.bigBuffer = make([]byte, defaultBufSize)
35+
newBuffer = p.bigBuffer[0:0]
36+
37+
//shift buffer back to the start of bigBuffer
38+
case 2*len(p.buffer) <= len(p.bigBuffer):
39+
newBuffer = p.bigBuffer[0:len(p.buffer)]
40+
41+
//reallocate big buffer with enough space to shift buffer
42+
default:
43+
p.bigBuffer = make([]byte, 2*len(p.buffer))
44+
newBuffer = p.bigBuffer[0:len(p.buffer)]
45+
}
46+
3047
copy(newBuffer, p.buffer)
3148
p.buffer = newBuffer
3249
}

‎parser_test.go‎

Lines changed: 75 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import (
44
"strings"
55
"testing"
66

7-
"github.com/stretchr/testify/assert"
7+
"github.com/stretchr/testify/suite"
88
)
99

1010
func BenchmarkParser_ReadMessage(b *testing.B) {
@@ -17,38 +17,40 @@ func BenchmarkParser_ReadMessage(b *testing.B) {
1717
}
1818
}
1919

20-
func TestParser_JumpLength(t *testing.T) {
21-
stream := "8=FIXT.1.19=11135=D34=449=TW52=20140511-23:10:3456=ISLD11=ID21=340=154=155=INTC60=20140511-23:10:3410=2348=FIXT.1.19=9535=D34=549=TW52=20140511-23:10:3456=ISLD11=ID21=340=154=155=INTC60=20140511-23:10:3410=198"
22-
23-
reader := strings.NewReader(stream)
24-
parser := newParser(reader)
25-
index, err := parser.jumpLength()
20+
type ParserSuite struct {
21+
suite.Suite
22+
*parser
23+
}
2624

27-
if err != nil {
28-
t.Error("Unexpected error: ", err)
29-
}
25+
func TestParserSuite(t *testing.T) {
26+
suite.Run(t, new(ParserSuite))
27+
}
3028

31-
expectedIndex := 111 + 17 - 1
32-
if index != expectedIndex {
33-
t.Error("expected index ", expectedIndex, " got ", index)
34-
}
29+
func (s *ParserSuite) SetupTest() {
30+
s.parser = new(parser)
3531
}
3632

37-
func TestParser_BadLength(t *testing.T) {
33+
func (s *ParserSuite) TestJumpLength() {
3834
stream := "8=FIXT.1.19=11135=D34=449=TW52=20140511-23:10:3456=ISLD11=ID21=340=154=155=INTC60=20140511-23:10:3410=2348=FIXT.1.19=9535=D34=549=TW52=20140511-23:10:3456=ISLD11=ID21=340=154=155=INTC60=20140511-23:10:3410=198"
3935

40-
reader := strings.NewReader(stream)
41-
parser := newParser(reader)
36+
s.reader = strings.NewReader(stream)
37+
index, err := s.parser.jumpLength()
38+
s.Nil(err)
4239

43-
bytes, _ := parser.ReadMessage()
40+
expectedIndex := 111 + 17 - 1
41+
s.Equal(expectedIndex, index)
42+
}
4443

45-
if stream != bytes.String() {
46-
t.Errorf("Expected %v, got %v", stream, bytes.String())
47-
}
44+
func (s *ParserSuite) TestBadLength() {
45+
stream := "8=FIXT.1.19=11135=D34=449=TW52=20140511-23:10:3456=ISLD11=ID21=340=154=155=INTC60=20140511-23:10:3410=2348=FIXT.1.19=9535=D34=549=TW52=20140511-23:10:3456=ISLD11=ID21=340=154=155=INTC60=20140511-23:10:3410=198"
46+
47+
s.reader = strings.NewReader(stream)
48+
bytes, _ := s.parser.ReadMessage()
4849

50+
s.Equal(stream, bytes.String())
4951
}
5052

51-
func TestParser_findStart(t *testing.T) {
53+
func (s *ParserSuite) TestFindStart() {
5254
var testCases = []struct {
5355
stream string
5456
expectError bool
@@ -59,31 +61,22 @@ func TestParser_findStart(t *testing.T) {
5961
{stream: "hello8=FIX.4.0", expectedStart: 5},
6062
}
6163
for _, tc := range testCases {
62-
reader := strings.NewReader(tc.stream)
63-
parser := newParser(reader)
64+
s.SetupTest()
6465

65-
start, err := parser.findStart()
66+
s.reader = strings.NewReader(tc.stream)
6667

67-
if err != nil {
68-
if !tc.expectError {
69-
t.Error("unxpected error", err)
70-
}
68+
start, err := s.parser.findStart()
69+
if tc.expectError {
70+
s.NotNil(err)
7171
continue
7272
}
7373

74-
if err == nil && tc.expectError {
75-
t.Error("expected error")
76-
}
77-
78-
if start != tc.expectedStart {
79-
t.Errorf("Expected start to be %v, but was %v", tc.expectedStart, start)
80-
}
74+
s.Nil(err)
75+
s.Equal(tc.expectedStart, start)
8176
}
82-
8377
}
8478

85-
func TestParser_ReadEOF(t *testing.T) {
86-
79+
func (s *ParserSuite) TestReadEOF() {
8780
var testCases = []struct {
8881
stream string
8982
}{
@@ -92,21 +85,20 @@ func TestParser_ReadEOF(t *testing.T) {
9285
}
9386

9487
for _, tc := range testCases {
95-
reader := strings.NewReader(tc.stream)
96-
parser := newParser(reader)
97-
bytes, err := parser.ReadMessage()
98-
assert.Nil(t, bytes)
88+
s.SetupTest()
9989

100-
if err == nil || err.Error() != "EOF" {
101-
t.Error("Expected EOF")
102-
}
90+
s.reader = strings.NewReader(tc.stream)
91+
bytes, err := s.parser.ReadMessage()
92+
s.Nil(bytes)
93+
94+
s.NotNil(err)
95+
s.Equal("EOF", err.Error())
10396
}
10497
}
10598

106-
func TestParser_ReadMessage(t *testing.T) {
99+
func (s *ParserSuite) TestReadMessage() {
107100
stream := "hello8=FIX.4.09=5blah10=1038=FIX.4.09=4foo10=103"
108-
reader := strings.NewReader(stream)
109-
parser := newParser(reader)
101+
s.reader = strings.NewReader(stream)
110102

111103
var testCases = []struct {
112104
expectedBytes string
@@ -118,68 +110,50 @@ func TestParser_ReadMessage(t *testing.T) {
118110
}
119111

120112
for _, tc := range testCases {
121-
msg, err := parser.ReadMessage()
122-
123-
if err != nil {
124-
t.Error("Unexpected error", err)
125-
}
126-
127-
if tc.expectedBytes != msg.String() {
128-
t.Errorf("Expected '%v' got '%v'", tc.expectedBytes, msg.String())
129-
}
130-
131-
if cap(parser.buffer) != tc.expectedBufferCap {
132-
t.Errorf("Expected capacity %v got %v", tc.expectedBufferCap, cap(parser.buffer))
133-
}
134-
135-
if len(parser.buffer) != tc.expectedBufferLen {
136-
t.Errorf("Expected len %v got %v", tc.expectedBufferLen, len(parser.buffer))
137-
}
113+
msg, err := s.parser.ReadMessage()
114+
s.Nil(err)
138115

116+
s.Equal(tc.expectedBytes, msg.String())
117+
s.Equal(tc.expectedBufferCap, cap(s.parser.buffer))
118+
s.Equal(tc.expectedBufferLen, len(s.parser.buffer))
139119
}
140120
}
141121

142-
func TestParser_ReadMessageGrowBuffer(t *testing.T) {
122+
func (s *ParserSuite) TestReadMessageGrowBuffer() {
143123
stream := "hello8=FIX.4.09=5blah10=1038=FIX.4.09=4foo10=103"
144124

145125
var testCases = []struct {
146-
initialBufCap int
147-
expectedBytes string
148-
expectedBufferLen int
149-
expectedBufferCap int
126+
initialBufCap int
127+
expectedBytes string
128+
expectedBufferLen int
129+
expectedBufferCap int
130+
expectedBigBufferLen int
150131
}{
151-
{initialBufCap: 0, expectedBufferCap: defaultBufSize - 31, expectedBufferLen: len(stream) - 31},
152-
{initialBufCap: 4, expectedBufferCap: defaultBufSize - 27, expectedBufferLen: len(stream) - 31},
153-
{initialBufCap: 8, expectedBufferCap: defaultBufSize - 23, expectedBufferLen: len(stream) - 31},
154-
{initialBufCap: 14, expectedBufferCap: defaultBufSize - 17, expectedBufferLen: len(stream) - 31},
155-
{initialBufCap: 16, expectedBufferCap: defaultBufSize - 15, expectedBufferLen: len(stream) - 31},
156-
{initialBufCap: 23, expectedBufferCap: defaultBufSize - 8, expectedBufferLen: len(stream) - 31},
157-
{initialBufCap: 30, expectedBufferCap: defaultBufSize - 1, expectedBufferLen: len(stream) - 31},
158-
{initialBufCap: 31, expectedBufferCap: 0, expectedBufferLen: 0},
159-
{initialBufCap: 40, expectedBufferCap: 9, expectedBufferLen: 9},
160-
{initialBufCap: 60, expectedBufferCap: 29, expectedBufferLen: 25},
161-
{initialBufCap: 80, expectedBufferCap: 49, expectedBufferLen: 25},
132+
{initialBufCap: 0, expectedBufferCap: defaultBufSize - 31, expectedBufferLen: len(stream) - 31, expectedBigBufferLen: defaultBufSize},
133+
{initialBufCap: 4, expectedBufferCap: 6, expectedBufferLen: 6, expectedBigBufferLen: 32},
134+
{initialBufCap: 8, expectedBufferCap: 6, expectedBufferLen: 6, expectedBigBufferLen: 32},
135+
{initialBufCap: 14, expectedBufferCap: 10, expectedBufferLen: 10, expectedBigBufferLen: 36},
136+
{initialBufCap: 16, expectedBufferCap: 18, expectedBufferLen: 18, expectedBigBufferLen: 44},
137+
{initialBufCap: 23, expectedBufferCap: 10, expectedBufferLen: 10, expectedBigBufferLen: 36},
138+
{initialBufCap: 30, expectedBufferCap: 24, expectedBufferLen: 24, expectedBigBufferLen: 50},
139+
{initialBufCap: 31, expectedBufferCap: 0, expectedBufferLen: 0, expectedBigBufferLen: 31},
140+
{initialBufCap: 40, expectedBufferCap: 9, expectedBufferLen: 9, expectedBigBufferLen: 40},
141+
{initialBufCap: 60, expectedBufferCap: 29, expectedBufferLen: 25, expectedBigBufferLen: 60},
142+
{initialBufCap: 80, expectedBufferCap: 49, expectedBufferLen: 25, expectedBigBufferLen: 80},
162143
}
163144

164145
for _, tc := range testCases {
165-
parser := newParser(strings.NewReader(stream))
166-
parser.buffer = make([]byte, 0, tc.initialBufCap)
167-
msg, err := parser.ReadMessage()
168-
169-
if err != nil {
170-
t.Fatal("unexpected err: ", err)
171-
}
172-
173-
if msg.String() != "8=FIX.4.09=5blah10=103" {
174-
t.Error("Didn't get expected message, got ", msg.String())
175-
}
176-
177-
if cap(parser.buffer) != tc.expectedBufferCap {
178-
t.Errorf("expected cap %v, got %v", tc.expectedBufferCap, cap(parser.buffer))
179-
}
180-
181-
if len(parser.buffer) != tc.expectedBufferLen {
182-
t.Errorf("expected len %v, got %v", tc.expectedBufferLen, len(parser.buffer))
183-
}
146+
s.SetupTest()
147+
148+
s.reader = strings.NewReader(stream)
149+
s.parser.bigBuffer = make([]byte, tc.initialBufCap)
150+
s.parser.buffer = s.parser.bigBuffer[0:0]
151+
msg, err := s.parser.ReadMessage()
152+
153+
s.Nil(err)
154+
s.Equal("8=FIX.4.09=5blah10=103", msg.String())
155+
s.Equal(tc.expectedBigBufferLen, len(s.parser.bigBuffer))
156+
s.Equal(tc.expectedBufferCap, cap(s.parser.buffer))
157+
s.Equal(tc.expectedBufferLen, len(s.parser.buffer))
184158
}
185159
}

0 commit comments

Comments
 (0)