Skip to content

Commit 62bbf8a

Browse files
committed
Merge pull request #1778 from fjl/rlp-trie-changes
rlp: precursor changes for trie, p2p
2 parents 90f1fe0 + fc8b246 commit 62bbf8a

File tree

6 files changed

+413
-11
lines changed

6 files changed

+413
-11
lines changed

rlp/decode.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,8 @@ var (
173173
func makeDecoder(typ reflect.Type, tags tags) (dec decoder, err error) {
174174
kind := typ.Kind()
175175
switch {
176+
case typ == rawValueType:
177+
return decodeRawValue, nil
176178
case typ.Implements(decoderInterface):
177179
return decodeDecoder, nil
178180
case kind != reflect.Ptr && reflect.PtrTo(typ).Implements(decoderInterface):
@@ -203,6 +205,15 @@ func makeDecoder(typ reflect.Type, tags tags) (dec decoder, err error) {
203205
}
204206
}
205207

208+
func decodeRawValue(s *Stream, val reflect.Value) error {
209+
r, err := s.Raw()
210+
if err != nil {
211+
return err
212+
}
213+
val.SetBytes(r)
214+
return nil
215+
}
216+
206217
func decodeUint(s *Stream, val reflect.Value) error {
207218
typ := val.Type()
208219
num, err := s.uint(typ.Bits())

rlp/decode_test.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"io"
2525
"math/big"
2626
"reflect"
27+
"strings"
2728
"testing"
2829
)
2930

@@ -437,6 +438,11 @@ var decodeTests = []decodeTest{
437438
error: "rlp: expected input string or byte for uint, decoding into (rlp.recstruct).Child.I",
438439
},
439440

441+
// RawValue
442+
{input: "01", ptr: new(RawValue), value: RawValue(unhex("01"))},
443+
{input: "82FFFF", ptr: new(RawValue), value: RawValue(unhex("82FFFF"))},
444+
{input: "C20102", ptr: new([]RawValue), value: []RawValue{unhex("01"), unhex("02")}},
445+
440446
// pointers
441447
{input: "00", ptr: new(*[]byte), value: &[]byte{0}},
442448
{input: "80", ptr: new(*uint), value: uintp(0)},
@@ -725,7 +731,7 @@ func encodeTestSlice(n uint) []byte {
725731
}
726732

727733
func unhex(str string) []byte {
728-
b, err := hex.DecodeString(str)
734+
b, err := hex.DecodeString(strings.Replace(str, " ", "", -1))
729735
if err != nil {
730736
panic(fmt.Sprintf("invalid hex string: %q", str))
731737
}

rlp/encode.go

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -45,12 +45,6 @@ type Encoder interface {
4545
EncodeRLP(io.Writer) error
4646
}
4747

48-
// ListSize returns the encoded size of an RLP list with the given
49-
// content size.
50-
func ListSize(contentSize uint64) uint64 {
51-
return uint64(headsize(contentSize)) + contentSize
52-
}
53-
5448
// Encode writes the RLP encoding of val to w. Note that Encode may
5549
// perform many small writes in some cases. Consider making w
5650
// buffered.
@@ -90,8 +84,8 @@ func Encode(w io.Writer, val interface{}) error {
9084
return outer.encode(val)
9185
}
9286
eb := encbufPool.Get().(*encbuf)
93-
eb.reset()
9487
defer encbufPool.Put(eb)
88+
eb.reset()
9589
if err := eb.encode(val); err != nil {
9690
return err
9791
}
@@ -102,8 +96,8 @@ func Encode(w io.Writer, val interface{}) error {
10296
// Please see the documentation of Encode for the encoding rules.
10397
func EncodeToBytes(val interface{}) ([]byte, error) {
10498
eb := encbufPool.Get().(*encbuf)
105-
eb.reset()
10699
defer encbufPool.Put(eb)
100+
eb.reset()
107101
if err := eb.encode(val); err != nil {
108102
return nil, err
109103
}
@@ -288,8 +282,13 @@ type encReader struct {
288282
func (r *encReader) Read(b []byte) (n int, err error) {
289283
for {
290284
if r.piece = r.next(); r.piece == nil {
291-
encbufPool.Put(r.buf)
292-
r.buf = nil
285+
// Put the encode buffer back into the pool at EOF when it
286+
// is first encountered. Subsequent calls still return EOF
287+
// as the error but the buffer is no longer valid.
288+
if r.buf != nil {
289+
encbufPool.Put(r.buf)
290+
r.buf = nil
291+
}
293292
return n, io.EOF
294293
}
295294
nn := copy(b[n:], r.piece)
@@ -349,6 +348,8 @@ var (
349348
func makeWriter(typ reflect.Type) (writer, error) {
350349
kind := typ.Kind()
351350
switch {
351+
case typ == rawValueType:
352+
return writeRawValue, nil
352353
case typ.Implements(encoderInterface):
353354
return writeEncoder, nil
354355
case kind != reflect.Ptr && reflect.PtrTo(typ).Implements(encoderInterface):
@@ -384,6 +385,11 @@ func isByte(typ reflect.Type) bool {
384385
return typ.Kind() == reflect.Uint8 && !typ.Implements(encoderInterface)
385386
}
386387

388+
func writeRawValue(val reflect.Value, w *encbuf) error {
389+
w.str = append(w.str, val.Bytes()...)
390+
return nil
391+
}
392+
387393
func writeUint(val reflect.Value, w *encbuf) error {
388394
i := val.Uint()
389395
if i == 0 {

rlp/encode_test.go

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"io"
2424
"io/ioutil"
2525
"math/big"
26+
"sync"
2627
"testing"
2728
)
2829

@@ -203,6 +204,11 @@ var encTests = []encTest{
203204
output: "F90200CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376",
204205
},
205206

207+
// RawValue
208+
{val: RawValue(unhex("01")), output: "01"},
209+
{val: RawValue(unhex("82FFFF")), output: "82FFFF"},
210+
{val: []RawValue{unhex("01"), unhex("02")}, output: "C20102"},
211+
206212
// structs
207213
{val: simplestruct{}, output: "C28080"},
208214
{val: simplestruct{A: 3, B: "foo"}, output: "C50383666F6F"},
@@ -306,3 +312,25 @@ func TestEncodeToReaderPiecewise(t *testing.T) {
306312
return output, nil
307313
})
308314
}
315+
316+
// This is a regression test verifying that encReader
317+
// returns its encbuf to the pool only once.
318+
func TestEncodeToReaderReturnToPool(t *testing.T) {
319+
buf := make([]byte, 50)
320+
wg := new(sync.WaitGroup)
321+
for i := 0; i < 5; i++ {
322+
wg.Add(1)
323+
go func() {
324+
for i := 0; i < 1000; i++ {
325+
_, r, _ := EncodeToReader("foo")
326+
ioutil.ReadAll(r)
327+
r.Read(buf)
328+
r.Read(buf)
329+
r.Read(buf)
330+
r.Read(buf)
331+
}
332+
wg.Done()
333+
}()
334+
}
335+
wg.Wait()
336+
}

rlp/raw.go

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
// Copyright 2015 The go-ethereum Authors
2+
// This file is part of the go-ethereum library.
3+
//
4+
// The go-ethereum library is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU Lesser General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// The go-ethereum library is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU Lesser General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU Lesser General Public License
15+
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
16+
17+
package rlp
18+
19+
import (
20+
"io"
21+
"reflect"
22+
)
23+
24+
// RawValue represents an encoded RLP value and can be used to delay
25+
// RLP decoding or precompute an encoding. Note that the decoder does
26+
// not verify whether the content of RawValues is valid RLP.
27+
type RawValue []byte
28+
29+
var rawValueType = reflect.TypeOf(RawValue{})
30+
31+
// ListSize returns the encoded size of an RLP list with the given
32+
// content size.
33+
func ListSize(contentSize uint64) uint64 {
34+
return uint64(headsize(contentSize)) + contentSize
35+
}
36+
37+
// Split returns the content of first RLP value and any
38+
// bytes after the value as subslices of b.
39+
func Split(b []byte) (k Kind, content, rest []byte, err error) {
40+
k, ts, cs, err := readKind(b)
41+
if err != nil {
42+
return 0, nil, b, err
43+
}
44+
return k, b[ts : ts+cs], b[ts+cs:], nil
45+
}
46+
47+
// SplitString splits b into the content of an RLP string
48+
// and any remaining bytes after the string.
49+
func SplitString(b []byte) (content, rest []byte, err error) {
50+
k, content, rest, err := Split(b)
51+
if err != nil {
52+
return nil, b, err
53+
}
54+
if k == List {
55+
return nil, b, ErrExpectedString
56+
}
57+
return content, rest, nil
58+
}
59+
60+
// SplitList splits b into the content of a list and any remaining
61+
// bytes after the list.
62+
func SplitList(b []byte) (content, rest []byte, err error) {
63+
k, content, rest, err := Split(b)
64+
if err != nil {
65+
return nil, b, err
66+
}
67+
if k != List {
68+
return nil, b, ErrExpectedList
69+
}
70+
return content, rest, nil
71+
}
72+
73+
// CountValues counts the number of encoded values in b.
74+
func CountValues(b []byte) (int, error) {
75+
i := 0
76+
for ; len(b) > 0; i++ {
77+
_, tagsize, size, err := readKind(b)
78+
if err != nil {
79+
return 0, err
80+
}
81+
b = b[tagsize+size:]
82+
}
83+
return i, nil
84+
}
85+
86+
func readKind(buf []byte) (k Kind, tagsize, contentsize uint64, err error) {
87+
if len(buf) == 0 {
88+
return 0, 0, 0, io.ErrUnexpectedEOF
89+
}
90+
b := buf[0]
91+
switch {
92+
case b < 0x80:
93+
k = Byte
94+
tagsize = 0
95+
contentsize = 1
96+
case b < 0xB8:
97+
k = String
98+
tagsize = 1
99+
contentsize = uint64(b - 0x80)
100+
// Reject strings that should've been single bytes.
101+
if contentsize == 1 && buf[1] < 128 {
102+
return 0, 0, 0, ErrCanonSize
103+
}
104+
case b < 0xC0:
105+
k = String
106+
tagsize = uint64(b-0xB7) + 1
107+
contentsize, err = readSize(buf[1:], b-0xB7)
108+
case b < 0xF8:
109+
k = List
110+
tagsize = 1
111+
contentsize = uint64(b - 0xC0)
112+
default:
113+
k = List
114+
tagsize = uint64(b-0xF7) + 1
115+
contentsize, err = readSize(buf[1:], b-0xF7)
116+
}
117+
if err != nil {
118+
return 0, 0, 0, err
119+
}
120+
// Reject values larger than the input slice.
121+
if contentsize > uint64(len(buf))-tagsize {
122+
return 0, 0, 0, ErrValueTooLarge
123+
}
124+
return k, tagsize, contentsize, err
125+
}
126+
127+
func readSize(b []byte, slen byte) (uint64, error) {
128+
if int(slen) > len(b) {
129+
return 0, io.ErrUnexpectedEOF
130+
}
131+
var s uint64
132+
switch slen {
133+
case 1:
134+
s = uint64(b[0])
135+
case 2:
136+
s = uint64(b[0])<<8 | uint64(b[1])
137+
case 3:
138+
s = uint64(b[0])<<16 | uint64(b[1])<<8 | uint64(b[2])
139+
case 4:
140+
s = uint64(b[0])<<24 | uint64(b[1])<<16 | uint64(b[2])<<8 | uint64(b[3])
141+
case 5:
142+
s = uint64(b[0])<<32 | uint64(b[1])<<24 | uint64(b[2])<<16 | uint64(b[3])<<8 | uint64(b[4])
143+
case 6:
144+
s = uint64(b[0])<<40 | uint64(b[1])<<32 | uint64(b[2])<<24 | uint64(b[3])<<16 | uint64(b[4])<<8 | uint64(b[5])
145+
case 7:
146+
s = uint64(b[0])<<48 | uint64(b[1])<<40 | uint64(b[2])<<32 | uint64(b[3])<<24 | uint64(b[4])<<16 | uint64(b[5])<<8 | uint64(b[6])
147+
case 8:
148+
s = uint64(b[0])<<56 | uint64(b[1])<<48 | uint64(b[2])<<40 | uint64(b[3])<<32 | uint64(b[4])<<24 | uint64(b[5])<<16 | uint64(b[6])<<8 | uint64(b[7])
149+
}
150+
// Reject sizes < 56 (shouldn't have separate size) and sizes with
151+
// leading zero bytes.
152+
if s < 56 || b[0] == 0 {
153+
return 0, ErrCanonSize
154+
}
155+
return s, nil
156+
}

0 commit comments

Comments
 (0)