Skip to content

Commit 25543dc

Browse files
author
roman.atachiants@careem.com
committedDec 5, 2021
Refactor readers
·
v1.0.19v1.0.16
1 parent 1f80b4d commit 25543dc

8 files changed

+363
-168
lines changed
 

‎codecs.go

+28-34
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ func (c *reflectSliceCodec) EncodeTo(e *Encoder, rv reflect.Value) (err error) {
7373
// Decode decodes into a reflect value from the decoder.
7474
func (c *reflectSliceCodec) DecodeTo(d *Decoder, rv reflect.Value) (err error) {
7575
var l uint64
76-
if l, err = binary.ReadUvarint(d.r); err == nil && l > 0 {
76+
if l, err = d.ReadUvarint(); err == nil && l > 0 {
7777
rv.Set(reflect.MakeSlice(rv.Type(), int(l), int(l)))
7878
for i := 0; i < int(l); i++ {
7979
v := reflect.Indirect(rv.Index(i))
@@ -112,7 +112,7 @@ func (c *reflectSliceOfPtrCodec) EncodeTo(e *Encoder, rv reflect.Value) (err err
112112
func (c *reflectSliceOfPtrCodec) DecodeTo(d *Decoder, rv reflect.Value) (err error) {
113113
var l uint64
114114
var isNil bool
115-
if l, err = binary.ReadUvarint(d.r); err == nil && l > 0 {
115+
if l, err = d.ReadUvarint(); err == nil && l > 0 {
116116
rv.Set(reflect.MakeSlice(rv.Type(), int(l), int(l)))
117117
for i := 0; i < int(l); i++ {
118118
if isNil, err = d.ReadBool(); !isNil {
@@ -175,7 +175,7 @@ func (c *boolSliceCodec) DecodeTo(d *Decoder, rv reflect.Value) (err error) {
175175
var l uint64
176176
if l, err = d.ReadUvarint(); err == nil && l > 0 {
177177
buf := make([]byte, l)
178-
_, err = d.r.Read(buf)
178+
_, err = d.Read(buf)
179179
rv.Set(reflect.ValueOf(binaryToBools(&buf)))
180180
}
181181
return
@@ -198,16 +198,14 @@ func (c *varintSliceCodec) EncodeTo(e *Encoder, rv reflect.Value) (err error) {
198198
// Decode decodes into a reflect value from the decoder.
199199
func (c *varintSliceCodec) DecodeTo(d *Decoder, rv reflect.Value) (err error) {
200200
var l uint64
201-
if l, err = binary.ReadUvarint(d.r); err == nil && l > 0 {
202-
slice := reflect.MakeSlice(rv.Type(), int(l), int(l))
201+
if l, err = d.ReadUvarint(); err == nil && l > 0 {
202+
rv.Set(reflect.MakeSlice(rv.Type(), int(l), int(l)))
203203
for i := 0; i < int(l); i++ {
204204
var v int64
205-
if v, err = binary.ReadVarint(d.r); err == nil {
206-
slice.Index(i).SetInt(v)
205+
if v, err = d.ReadVarint(); err == nil {
206+
rv.Index(i).SetInt(v)
207207
}
208208
}
209-
210-
rv.Set(slice)
211209
}
212210
return
213211
}
@@ -229,15 +227,13 @@ func (c *varuintSliceCodec) EncodeTo(e *Encoder, rv reflect.Value) (err error) {
229227
// Decode decodes into a reflect value from the decoder.
230228
func (c *varuintSliceCodec) DecodeTo(d *Decoder, rv reflect.Value) (err error) {
231229
var l, v uint64
232-
if l, err = binary.ReadUvarint(d.r); err == nil && l > 0 {
233-
slice := reflect.MakeSlice(rv.Type(), int(l), int(l))
230+
if l, err = d.ReadUvarint(); err == nil && l > 0 {
231+
rv.Set(reflect.MakeSlice(rv.Type(), int(l), int(l)))
234232
for i := 0; i < int(l); i++ {
235233
if v, err = d.ReadUvarint(); err == nil {
236-
slice.Index(i).SetUint(v)
234+
rv.Index(i).SetUint(v)
237235
}
238236
}
239-
240-
rv.Set(slice)
241237
}
242238
return
243239
}
@@ -290,8 +286,8 @@ type fieldCodec struct {
290286
}
291287

292288
// Encode encodes a value into the encoder.
293-
func (c *reflectStructCodec) EncodeTo(e *Encoder, rv reflect.Value) (err error) {
294-
for _, i := range *c {
289+
func (c reflectStructCodec) EncodeTo(e *Encoder, rv reflect.Value) (err error) {
290+
for _, i := range c {
295291
if err = i.Codec.EncodeTo(e, rv.Field(i.Index)); err != nil {
296292
return
297293
}
@@ -300,17 +296,18 @@ func (c *reflectStructCodec) EncodeTo(e *Encoder, rv reflect.Value) (err error)
300296
}
301297

302298
// Decode decodes into a reflect value from the decoder.
303-
func (c *reflectStructCodec) DecodeTo(d *Decoder, rv reflect.Value) (err error) {
304-
for _, i := range *c {
299+
func (c reflectStructCodec) DecodeTo(d *Decoder, rv reflect.Value) (err error) {
300+
for _, i := range c {
305301
v := rv.Field(i.Index)
306-
if v.Kind() == reflect.Ptr {
307-
if err = i.Codec.DecodeTo(d, v); err != nil {
308-
return
309-
}
310-
} else if v.CanSet() {
311-
if err = i.Codec.DecodeTo(d, reflect.Indirect(v)); err != nil {
312-
return
313-
}
302+
switch {
303+
case v.Kind() == reflect.Ptr:
304+
err = i.Codec.DecodeTo(d, v)
305+
case v.CanSet():
306+
err = i.Codec.DecodeTo(d, reflect.Indirect(v))
307+
}
308+
309+
if err != nil {
310+
return
314311
}
315312
}
316313
return
@@ -372,9 +369,9 @@ func (c *customCodec) DecodeTo(d *Decoder, rv reflect.Value) (err error) {
372369
}
373370

374371
var l uint64
375-
if l, err = binary.ReadUvarint(d.r); err == nil {
372+
if l, err = d.ReadUvarint(); err == nil {
376373
buffer := make([]byte, l)
377-
_, err = d.r.Read(buffer)
374+
_, err = d.Read(buffer)
378375
ret := m.Call([]reflect.Value{reflect.ValueOf(buffer)})
379376
if !ret[0].IsNil() {
380377
err = ret[0].Interface().(error)
@@ -463,21 +460,18 @@ func (c *reflectMapCodec) DecodeTo(d *Decoder, rv reflect.Value) (err error) {
463460
// Write key writes a key to the encoder
464461
func (c *reflectMapCodec) writeKey(e *Encoder, key reflect.Value) (err error) {
465462
switch key.Kind() {
466-
467463
case reflect.Int16:
468464
e.WriteUint16(uint16(key.Int()))
469465
case reflect.Int32:
470466
e.WriteUint32(uint32(key.Int()))
471467
case reflect.Int64:
472468
e.WriteUint64(uint64(key.Int()))
473-
474469
case reflect.Uint16:
475470
e.WriteUint16(uint16(key.Uint()))
476471
case reflect.Uint32:
477472
e.WriteUint32(uint32(key.Uint()))
478473
case reflect.Uint64:
479-
e.WriteUint64(uint64(key.Uint()))
480-
474+
e.WriteUint64(key.Uint())
481475
case reflect.String:
482476
str := key.String()
483477
e.WriteUint16(uint16(len(str)))
@@ -594,7 +588,7 @@ func (c *varintCodec) EncodeTo(e *Encoder, rv reflect.Value) error {
594588
// Decode decodes into a reflect value from the decoder.
595589
func (c *varintCodec) DecodeTo(d *Decoder, rv reflect.Value) (err error) {
596590
var v int64
597-
if v, err = binary.ReadVarint(d.r); err != nil {
591+
if v, err = d.ReadVarint(); err != nil {
598592
return
599593
}
600594
rv.SetInt(v)
@@ -614,7 +608,7 @@ func (c *varuintCodec) EncodeTo(e *Encoder, rv reflect.Value) error {
614608
// Decode decodes into a reflect value from the decoder.
615609
func (c *varuintCodec) DecodeTo(d *Decoder, rv reflect.Value) (err error) {
616610
var v uint64
617-
if v, err = binary.ReadUvarint(d.r); err != nil {
611+
if v, err = d.ReadUvarint(); err != nil {
618612
return
619613
}
620614
rv.SetUint(v)

‎codecs_test.go

+10
Original file line numberDiff line numberDiff line change
@@ -590,3 +590,13 @@ func TestSliceOfTimePtrs(t *testing.T) {
590590
assert.NoError(t, err)
591591
assert.Equal(t, v, o)
592592
}
593+
594+
func TestEncodeBigStruct(t *testing.T) {
595+
input := newBigStruct()
596+
b, err := Marshal(input)
597+
assert.NoError(t, err)
598+
599+
var output bigStruct
600+
assert.NoError(t, Unmarshal(b, &output))
601+
assert.Equal(t, input, &output)
602+
}

‎decoder.go

+16-51
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,12 @@ var decoders = &sync.Pool{New: func() interface{} {
1717
return NewDecoder(newReader(nil))
1818
}}
1919

20-
// Reader represents the interface a reader should implement.
21-
type Reader interface {
22-
io.Reader
23-
io.ByteReader
24-
}
25-
2620
// Unmarshal decodes the payload from the binary format.
2721
func Unmarshal(b []byte, v interface{}) (err error) {
2822

2923
// Get the decoder from the pool, reset it
3024
d := decoders.Get().(*Decoder)
31-
d.r.(*reader).Reset(b) // Reset the reader
25+
d.reader.(*sliceReader).Reset(b) // Reset the reader
3226

3327
// Decode and set the buffer if successful and free the decoder
3428
err = d.Decode(v)
@@ -38,22 +32,15 @@ func Unmarshal(b []byte, v interface{}) (err error) {
3832

3933
// Decoder represents a binary decoder.
4034
type Decoder struct {
41-
r Reader
42-
s *reader // Not using the interface for better inlining
35+
reader reader
4336
scratch [10]byte
4437
schemas map[reflect.Type]Codec
4538
}
4639

4740
// NewDecoder creates a binary decoder.
48-
func NewDecoder(r Reader) *Decoder {
49-
var slicer *reader
50-
if s, ok := r.(*reader); ok {
51-
slicer = s
52-
}
53-
41+
func NewDecoder(r io.Reader) *Decoder {
5442
return &Decoder{
55-
r: r,
56-
s: slicer,
43+
reader: newReader(r),
5744
schemas: make(map[reflect.Type]Codec),
5845
}
5946
}
@@ -62,7 +49,7 @@ func NewDecoder(r Reader) *Decoder {
6249
func (d *Decoder) Decode(v interface{}) (err error) {
6350
rv := reflect.Indirect(reflect.ValueOf(v))
6451
if !rv.CanAddr() {
65-
return errors.New("binary: can only Decode to pointer type")
52+
return errors.New("binary: can only decode to pointer type")
6653
}
6754

6855
// Scan the type (this will load from cache)
@@ -76,23 +63,23 @@ func (d *Decoder) Decode(v interface{}) (err error) {
7663

7764
// Read reads a set of bytes
7865
func (d *Decoder) Read(b []byte) (int, error) {
79-
return d.r.Read(b)
66+
return d.reader.Read(b)
8067
}
8168

8269
// ReadUvarint reads a variable-length Uint64 from the buffer.
8370
func (d *Decoder) ReadUvarint() (uint64, error) {
84-
return binary.ReadUvarint(d.r)
71+
return d.reader.ReadUvarint()
8572
}
8673

8774
// ReadVarint reads a variable-length Int64 from the buffer.
8875
func (d *Decoder) ReadVarint() (int64, error) {
89-
return binary.ReadVarint(d.r)
76+
return d.reader.ReadVarint()
9077
}
9178

9279
// ReadUint16 reads a uint16
9380
func (d *Decoder) ReadUint16() (out uint16, err error) {
9481
var b []byte
95-
if b, err = d.sliceOrScratch(2); err == nil {
82+
if b, err = d.reader.Slice(2); err == nil {
9683
_ = b[1] // bounds check hint to compiler
9784
out = (uint16(b[0]) | uint16(b[1])<<8)
9885
}
@@ -102,7 +89,7 @@ func (d *Decoder) ReadUint16() (out uint16, err error) {
10289
// ReadUint32 reads a uint32
10390
func (d *Decoder) ReadUint32() (out uint32, err error) {
10491
var b []byte
105-
if b, err = d.sliceOrScratch(4); err == nil {
92+
if b, err = d.reader.Slice(4); err == nil {
10693
_ = b[3] // bounds check hint to compiler
10794
out = (uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24)
10895
}
@@ -112,7 +99,7 @@ func (d *Decoder) ReadUint32() (out uint32, err error) {
11299
// ReadUint64 reads a uint64
113100
func (d *Decoder) ReadUint64() (out uint64, err error) {
114101
var b []byte
115-
if b, err = d.sliceOrScratch(8); err == nil {
102+
if b, err = d.reader.Slice(8); err == nil {
116103
_ = b[7] // bounds check hint to compiler
117104
out = (uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 |
118105
uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56)
@@ -140,7 +127,7 @@ func (d *Decoder) ReadFloat64() (out float64, err error) {
140127

141128
// ReadBool reads a single boolean value from the slice.
142129
func (d *Decoder) ReadBool() (bool, error) {
143-
b, err := d.r.ReadByte()
130+
b, err := d.reader.ReadByte()
144131
return b == 1, err
145132
}
146133

@@ -155,44 +142,22 @@ func (d *Decoder) ReadString() (out string, err error) {
155142

156143
// ReadComplex reads a complex64
157144
func (d *Decoder) readComplex64() (out complex64, err error) {
158-
err = binary.Read(d.r, binary.LittleEndian, &out)
145+
err = binary.Read(d.reader, binary.LittleEndian, &out)
159146
return
160147
}
161148

162149
// ReadComplex reads a complex128
163150
func (d *Decoder) readComplex128() (out complex128, err error) {
164-
err = binary.Read(d.r, binary.LittleEndian, &out)
165-
return
166-
}
167-
168-
// sliceOrScratch a slice or reads into as scratch buffer. This is useful for values
169-
// which will get reallocated after this, such as ints, floats, etc.
170-
func (d *Decoder) sliceOrScratch(n int) (buffer []byte, err error) {
171-
if d.s != nil {
172-
return d.s.Slice(n)
173-
}
174-
175-
buffer = d.scratch[:n]
176-
_, err = d.r.Read(buffer)
151+
err = binary.Read(d.reader, binary.LittleEndian, &out)
177152
return
178153
}
179154

180155
// Slice selects a sub-slice of next bytes. This is similar to Read() but does not
181156
// actually perform a copy, but simply uses the underlying slice (if available) and
182157
// returns a sub-slice pointing to the same array. Since this requires access
183-
// to the underlying data, this is only available for our default reader.
158+
// to the underlying data, this is only available for a slice reader.
184159
func (d *Decoder) Slice(n int) ([]byte, error) {
185-
if d.s != nil {
186-
return d.s.Slice(n)
187-
}
188-
189-
// If we don't have a slicer, we can just allocate and read
190-
buffer := make([]byte, n, n)
191-
if _, err := d.Read(buffer); err != nil {
192-
return nil, err
193-
}
194-
195-
return buffer, nil
160+
return d.reader.Slice(n)
196161
}
197162

198163
// ReadSlice reads a varint prefixed sub-slice without copying and returns the underlying

‎encoder_test.go

+13-44
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ package binary
55

66
import (
77
"bytes"
8-
"encoding/gob"
98
"encoding/json"
109
"testing"
1110
"unsafe"
@@ -70,15 +69,6 @@ func Test_Full(t *testing.T) {
7069
assert.Equal(t, v, o)
7170
}
7271

73-
//type benchStruct = composite
74-
75-
type benchStruct = msg
76-
77-
func newBenchStruct() benchStruct {
78-
//return newComposite()
79-
return testMsg
80-
}
81-
8272
func newComposite() composite {
8373
v := composite{}
8474
v["a"] = column{
@@ -97,11 +87,14 @@ func newComposite() composite {
9787
return v
9888
}
9989

100-
// Benchmark_Binary/marshal-8 5286771 226 ns/op 112 B/op 2 allocs/op
101-
// Benchmark_Binary/marshal-to-8 6467770 167 ns/op 33 B/op 0 allocs/op
102-
// Benchmark_Binary/unmarshal-8 3350119 355 ns/op 88 B/op 5 allocs/op
103-
func Benchmark_Binary(b *testing.B) {
104-
v := newBenchStruct()
90+
/*
91+
cpu: Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz
92+
Benchmark_Binary/marshal-12 5074890 227.4 ns/op 112 B/op 2 allocs/op
93+
Benchmark_Binary/marshal-to-12 7011523 162.3 ns/op 30 B/op 0 allocs/op
94+
Benchmark_Binary/unmarshal-12 4224048 283.0 ns/op 72 B/op 5 allocs/op
95+
*/
96+
func BenchmarkBinary(b *testing.B) {
97+
v := testMsg
10598
enc, _ := Marshal(&v)
10699

107100
b.Run("marshal", func(b *testing.B) {
@@ -117,47 +110,23 @@ func Benchmark_Binary(b *testing.B) {
117110
b.ReportAllocs()
118111
b.ResetTimer()
119112
for n := 0; n < b.N; n++ {
113+
buffer.Reset()
120114
MarshalTo(&v, &buffer)
121115
}
122116
})
123117

124118
b.Run("unmarshal", func(b *testing.B) {
125119
b.ReportAllocs()
126120
b.ResetTimer()
127-
var out benchStruct
121+
var out msg
128122
for n := 0; n < b.N; n++ {
129123
Unmarshal(enc, &out)
130124
}
131125
})
132126
}
133127

134-
func Benchmark_Gob(b *testing.B) {
135-
v := newBenchStruct()
136-
137-
buffer := new(bytes.Buffer)
138-
codec := gob.NewEncoder(buffer)
139-
codec.Encode(&v)
140-
141-
b.Run("marshal", func(b *testing.B) {
142-
b.ReportAllocs()
143-
b.ResetTimer()
144-
for n := 0; n < b.N; n++ {
145-
gob.NewEncoder(new(bytes.Buffer)).Encode(&v)
146-
}
147-
})
148-
149-
b.Run("unmarshal", func(b *testing.B) {
150-
b.ReportAllocs()
151-
b.ResetTimer()
152-
var out benchStruct
153-
for n := 0; n < b.N; n++ {
154-
gob.NewDecoder(buffer).Decode(&out)
155-
}
156-
})
157-
}
158-
159-
func Benchmark_JSON(b *testing.B) {
160-
v := newBenchStruct()
128+
func BenchmarkJSON(b *testing.B) {
129+
v := testMsg
161130
enc, _ := json.Marshal(&v)
162131

163132
b.Run("marshal", func(b *testing.B) {
@@ -171,7 +140,7 @@ func Benchmark_JSON(b *testing.B) {
171140
b.Run("unmarshal", func(b *testing.B) {
172141
b.ReportAllocs()
173142
b.ResetTimer()
174-
var out benchStruct
143+
var out msg
175144
for n := 0; n < b.N; n++ {
176145
json.Unmarshal(enc, &out)
177146
}

‎go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
module github.com/kelindar/binary
22

3-
go 1.16
3+
go 1.17
44

55
require (
66
github.com/davecgh/go-spew v1.1.1 // indirect

‎reader.go

+146-30
Original file line numberDiff line numberDiff line change
@@ -9,76 +9,192 @@
99
package binary
1010

1111
import (
12+
"bufio"
13+
"bytes"
14+
"encoding/binary"
15+
"errors"
1216
"io"
1317
)
1418

15-
// A Reader implements the io.Reader, io.ReaderAt, io.WriterTo, io.Seeker,
16-
// io.ByteScanner, and io.RuneScanner interfaces by reading from
17-
// a byte slice.
18-
// Unlike a Buffer, a Reader is read-only and supports seeking.
19-
type reader struct {
20-
s []byte
21-
i int64 // current reading index
19+
// MaxVarintLenN is the maximum length of a varint-encoded N-bit integer.
20+
const (
21+
maxVarintLen64 = 10 * 7
22+
)
23+
24+
var overflow = errors.New("binary: varint overflows a 64-bit integer")
25+
26+
// reader represents a required contract for a decoder to work properly
27+
type reader interface {
28+
io.Reader
29+
io.ByteReader
30+
Slice(n int) (buffer []byte, err error)
31+
ReadUvarint() (uint64, error)
32+
ReadVarint() (int64, error)
33+
}
34+
35+
// newReader figures out the most efficient reader to use for the provided type
36+
func newReader(r io.Reader) reader {
37+
switch v := r.(type) {
38+
case nil:
39+
return newSliceReader(nil)
40+
case *bytes.Buffer:
41+
return newSliceReader(v.Bytes())
42+
case *sliceReader:
43+
return v
44+
default:
45+
rdr, ok := r.(reader)
46+
if !ok {
47+
rdr = newStreamReader(r)
48+
}
49+
return rdr
50+
}
51+
}
52+
53+
// --------------------------------------- Slice Reader ---------------------------------------
54+
55+
// sliceReader implements a reader that only reads from a slice
56+
type sliceReader struct {
57+
buffer []byte
58+
offset int64 // current reading index
59+
}
60+
61+
// newSliceReader returns a new Reader reading from b.
62+
func newSliceReader(b []byte) *sliceReader {
63+
return &sliceReader{b, 0}
2264
}
2365

2466
// Len returns the number of bytes of the unread portion of the
2567
// slice.
26-
func (r *reader) Len() int {
27-
if r.i >= int64(len(r.s)) {
68+
func (r *sliceReader) Len() int {
69+
if r.offset >= int64(len(r.buffer)) {
2870
return 0
2971
}
30-
return int(int64(len(r.s)) - r.i)
72+
return int(int64(len(r.buffer)) - r.offset)
3173
}
3274

3375
// Size returns the original length of the underlying byte slice.
3476
// Size is the number of bytes available for reading via ReadAt.
3577
// The returned value is always the same and is not affected by calls
3678
// to any other method.
37-
func (r *reader) Size() int64 { return int64(len(r.s)) }
79+
func (r *sliceReader) Size() int64 { return int64(len(r.buffer)) }
3880

3981
// Read implements the io.Reader interface.
40-
func (r *reader) Read(b []byte) (n int, err error) {
41-
if r.i >= int64(len(r.s)) {
82+
func (r *sliceReader) Read(b []byte) (n int, err error) {
83+
if r.offset >= int64(len(r.buffer)) {
4284
return 0, io.EOF
4385
}
4486

45-
n = copy(b, r.s[r.i:])
46-
r.i += int64(n)
87+
n = copy(b, r.buffer[r.offset:])
88+
r.offset += int64(n)
4789
return
4890
}
4991

5092
// ReadByte implements the io.ByteReader interface.
51-
func (r *reader) ReadByte() (byte, error) {
52-
if r.i >= int64(len(r.s)) {
93+
func (r *sliceReader) ReadByte() (byte, error) {
94+
if r.offset >= int64(len(r.buffer)) {
5395
return 0, io.EOF
5496
}
5597

56-
b := r.s[r.i]
57-
r.i++
98+
b := r.buffer[r.offset]
99+
r.offset++
58100
return b, nil
59101
}
60102

61103
// Slice selects a sub-slice of next bytes. This is similar to Read() but does not
62104
// actually perform a copy, but simply uses the underlying slice (if available) and
63105
// returns a sub-slice pointing to the same array. Since this requires access
64106
// to the underlying data, this is only available for our default reader.
65-
func (r *reader) Slice(n int) ([]byte, error) {
66-
if r.i+int64(n) > int64(len(r.s)) {
107+
func (r *sliceReader) Slice(n int) ([]byte, error) {
108+
if r.offset+int64(n) > int64(len(r.buffer)) {
67109
return nil, io.EOF
68110
}
69111

70-
cur := r.i
71-
r.i += int64(n)
72-
return r.s[cur:r.i], nil
112+
cur := r.offset
113+
r.offset += int64(n)
114+
return r.buffer[cur:r.offset], nil
115+
}
116+
117+
// ReadUvarint reads an encoded unsigned integer from r and returns it as a uint64.
118+
func (r *sliceReader) ReadUvarint() (uint64, error) {
119+
var x uint64
120+
for s := 0; s < maxVarintLen64; s += 7 {
121+
if r.offset >= int64(len(r.buffer)) {
122+
return 0, io.EOF
123+
}
124+
125+
b := r.buffer[r.offset]
126+
r.offset++
127+
if b < 0x80 {
128+
if s == maxVarintLen64-7 && b > 1 {
129+
return x, overflow
130+
}
131+
return x | uint64(b)<<s, nil
132+
}
133+
x |= uint64(b&0x7f) << s
134+
}
135+
return x, overflow
136+
}
137+
138+
// ReadVarint reads an encoded signed integer from r and returns it as an int64.
139+
func (r *sliceReader) ReadVarint() (int64, error) {
140+
ux, err := r.ReadUvarint() // ok to continue in presence of error
141+
x := int64(ux >> 1)
142+
if ux&1 != 0 {
143+
x = ^x
144+
}
145+
return x, err
73146
}
74147

75148
// Reset resets the Reader to be reading from b.
76-
func (r *reader) Reset(b []byte) {
77-
r.s = b
78-
r.i = 0
149+
func (r *sliceReader) Reset(b []byte) {
150+
r.buffer = b
151+
r.offset = 0
152+
}
153+
154+
// --------------------------------------- Stream Reader ---------------------------------------
155+
156+
// streamReader represents a reader implementation for a generic reader (i.e. streams)
157+
type streamReader struct {
158+
Reader
159+
scratch [10]byte
160+
}
161+
162+
// Reader represents the interface a reader should implement.
163+
type Reader interface {
164+
io.Reader
165+
io.ByteReader
166+
}
167+
168+
// newStreamReader returns a new stream reader
169+
func newStreamReader(r io.Reader) *streamReader {
170+
rdr, ok := r.(Reader)
171+
if !ok {
172+
rdr = bufio.NewReader(r)
173+
}
174+
175+
return &streamReader{
176+
Reader: rdr,
177+
}
178+
}
179+
180+
// Slice selects a sub-slice of next bytes.
181+
func (r *streamReader) Slice(n int) (buffer []byte, err error) {
182+
if n <= 10 {
183+
buffer = r.scratch[:n]
184+
} else {
185+
buffer = make([]byte, n, n)
186+
}
187+
188+
_, err = r.Read(buffer)
189+
return
190+
}
191+
192+
// ReadUvarint reads an encoded unsigned integer from r and returns it as a uint64.
193+
func (r *streamReader) ReadUvarint() (uint64, error) {
194+
return binary.ReadUvarint(r)
79195
}
80196

81-
// newReader returns a new Reader reading from b.
82-
func newReader(b []byte) *reader {
83-
return &reader{b, 0}
197+
// ReadVarint reads a variable-length Int64 from the buffer.
198+
func (r *streamReader) ReadVarint() (int64, error) {
199+
return binary.ReadVarint(r)
84200
}

‎reader_test.go

+141-1
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,156 @@
44
package binary
55

66
import (
7+
"bytes"
8+
"io"
9+
"math"
710
"testing"
11+
"time"
812

913
"github.com/stretchr/testify/assert"
1014
)
1115

1216
func TestReader_Slice(t *testing.T) {
13-
r := newReader([]byte("0123456789"))
17+
r := newSliceReader([]byte("0123456789"))
1418

1519
out, err := r.Slice(3)
1620
assert.NoError(t, err)
1721
assert.Len(t, out, 3)
1822
assert.Equal(t, "012", string(out))
23+
assert.Equal(t, 7, r.Len())
24+
}
25+
26+
func TestReaderEOF(t *testing.T) {
27+
b, _ := Marshal(newBigStruct())
28+
29+
for size := 0; size < len(b)-1; size++ {
30+
var output bigStruct
31+
assert.Error(t, Unmarshal(b[0:size], &output))
32+
}
33+
}
34+
35+
func TestStreamReader(t *testing.T) {
36+
input := newBigStruct()
37+
b, _ := Marshal(input)
38+
39+
dec := NewDecoder(newNetworkSource(b))
40+
out := new(bigStruct)
41+
assert.NoError(t, dec.Decode(out))
42+
}
43+
44+
// --------------------------------------- Big Structure (Every Field Type) ---------------------------------------
45+
46+
// structure with every possible codec type
47+
type bigStruct struct {
48+
String string
49+
Uint8 uint8
50+
Uint16 uint16
51+
Uint32 uint32
52+
Uint64 uint64
53+
Int8 int8
54+
Int16 int16
55+
Int32 int32
56+
Int64 int64
57+
Float32 float32
58+
Float64 float64
59+
Strings []string
60+
Bytes []byte
61+
Bools []bool
62+
Uint8s []uint8
63+
Uint16s []uint16
64+
Uint32s []uint32
65+
Uint64s []uint64
66+
Int8s []int8
67+
Int16s []int16
68+
Int32s []int32
69+
Int64s []int64
70+
Float32s []float32
71+
Float64s []float64
72+
MapStr map[string]simpleStruct
73+
MapPtr map[string]*simpleStruct
74+
MapUint8 map[uint8]uint8
75+
MapUint16 map[uint16]uint16
76+
MapUint32 map[uint32]uint32
77+
MapUint64 map[uint64]uint64
78+
MapInt8 map[int8]int8
79+
MapInt16 map[int16]int16
80+
MapInt32 map[int32]int32
81+
MapInt64 map[int64]int64
82+
Time *time.Time
83+
Nil *time.Time
84+
Pointer *simpleStruct
85+
Value simpleStruct
86+
Array [6]byte
87+
Byte byte
88+
Bool bool
89+
}
90+
91+
func newBigStruct() *bigStruct {
92+
timestamp := time.Date(2013, 1, 2, 3, 4, 5, 6, time.UTC)
93+
child := simpleStruct{
94+
Name: "Roman",
95+
Timestamp: timestamp,
96+
Payload: []byte("hi"),
97+
Ssid: []uint32{1, 2, 3},
98+
}
99+
100+
return &bigStruct{
101+
String: "hello",
102+
Byte: 0x3a,
103+
Bool: true,
104+
Uint8: math.MaxUint8,
105+
Uint16: math.MaxUint16,
106+
Uint32: math.MaxUint32,
107+
Uint64: math.MaxUint64,
108+
Int8: math.MaxInt8,
109+
Int16: math.MaxInt16,
110+
Int32: math.MaxInt32,
111+
Int64: math.MaxInt64,
112+
Float32: math.MaxFloat32,
113+
Float64: math.MaxFloat64,
114+
Strings: []string{"a", "b", "c"},
115+
Bytes: []byte("hello-bytes"),
116+
Bools: []bool{true, false, true},
117+
Uint8s: []uint8{0, math.MaxUint8},
118+
Uint16s: []uint16{0, math.MaxUint16},
119+
Uint32s: []uint32{0, math.MaxUint32},
120+
Uint64s: []uint64{0, math.MaxUint64},
121+
Int8s: []int8{math.MinInt8, math.MaxInt8},
122+
Int16s: []int16{math.MinInt16, math.MaxInt16},
123+
Int32s: []int32{math.MinInt32, math.MaxInt32},
124+
Int64s: []int64{math.MinInt64, math.MaxInt64},
125+
Float32s: []float32{0, math.MaxFloat32},
126+
Float64s: []float64{0, math.MaxFloat64},
127+
MapStr: map[string]simpleStruct{"a": child, "b": child},
128+
MapPtr: map[string]*simpleStruct{"a": &child, "b": nil},
129+
MapUint8: map[uint8]uint8{1: 1},
130+
MapUint16: map[uint16]uint16{1: 1},
131+
MapUint32: map[uint32]uint32{1: 1},
132+
MapUint64: map[uint64]uint64{1: 1},
133+
MapInt8: map[int8]int8{1: 1},
134+
MapInt16: map[int16]int16{1: 1},
135+
MapInt32: map[int32]int32{1: 1},
136+
MapInt64: map[int64]int64{1: 1},
137+
Array: [6]byte{1, 2, 3, 4, 5, 6},
138+
Time: &timestamp,
139+
Nil: nil,
140+
Pointer: &child,
141+
Value: child,
142+
}
143+
}
144+
145+
// --------------------------------------- Fake Network Reader ---------------------------------------
146+
147+
type networkSource struct {
148+
r io.Reader
149+
}
150+
151+
func newNetworkSource(data []byte) io.Reader {
152+
return &networkSource{
153+
r: bytes.NewBuffer(data),
154+
}
155+
}
156+
157+
func (s *networkSource) Read(p []byte) (n int, err error) {
158+
return s.r.Read(p)
19159
}

‎scanner.go

+8-7
Original file line numberDiff line numberDiff line change
@@ -131,17 +131,18 @@ func scanType(t reflect.Type) (Codec, error) {
131131

132132
case reflect.Struct:
133133
s := scanStruct(t)
134-
var v reflectStructCodec
134+
v := make(reflectStructCodec, len(s.fields))
135135
for _, i := range s.fields {
136136
field := t.Field(i)
137-
if c, err := scanType(field.Type); err == nil {
138-
v = append(v, fieldCodec{
139-
Index: i,
140-
Codec: c,
141-
})
142-
} else {
137+
codec, err := scanType(field.Type)
138+
if err != nil {
143139
return nil, err
144140
}
141+
142+
v[i] = fieldCodec{
143+
Index: i,
144+
Codec: codec,
145+
}
145146
}
146147

147148
return &v, nil

0 commit comments

Comments
 (0)
Please sign in to comment.