encoding/gob: reduce the amount of memory allocations.
Benchmark results:
benchmark old ns/op new ns/op delta
BenchmarkEndToEndPipe-4 7547 7294 -3.35%
BenchmarkEndToEndByteBuffer-4 5146 5092 -1.05%
BenchmarkEndToEndSliceByteBuffer-4 552779 439768 -20.44%
BenchmarkEncodeComplex128Slice-4 266370 266184 -0.07%
BenchmarkEncodeFloat64Slice-4 111891 110258 -1.46%
BenchmarkEncodeInt32Slice-4 74482 74080 -0.54%
BenchmarkEncodeStringSlice-4 84404 84279 -0.15%
BenchmarkEncodeInterfaceSlice-4 3942925 3045995 -22.75%
BenchmarkDecodeComplex128Slice-4 451837 415282 -8.09%
BenchmarkDecodeFloat64Slice-4 283584 262558 -7.41%
BenchmarkDecodeInt32Slice-4 246571 237383 -3.73%
BenchmarkDecodeStringSlice-4 734210 479625 -34.67%
BenchmarkDecodeInterfaceSlice-4 4778225 4160935 -12.92%
benchmark old allocs new allocs delta
BenchmarkEndToEndPipe-4 3 2 -33.33%
BenchmarkEndToEndByteBuffer-4 3 2 -33.33%
BenchmarkEndToEndSliceByteBuffer-4 1002 402 -59.88%
BenchmarkEncodeComplex128Slice-4 1 1 +0.00%
BenchmarkEncodeFloat64Slice-4 1 1 +0.00%
BenchmarkEncodeInt32Slice-4 1 1 +0.00%
BenchmarkEncodeStringSlice-4 1 1 +0.00%
BenchmarkEncodeInterfaceSlice-4 3001 1 -99.97%
BenchmarkDecodeComplex128Slice-4 188 185 -1.60%
BenchmarkDecodeFloat64Slice-4 188 185 -1.60%
BenchmarkDecodeInt32Slice-4 188 185 -1.60%
BenchmarkDecodeStringSlice-4 2188 1185 -45.84%
BenchmarkDecodeInterfaceSlice-4 6197 4194 -32.32%
benchmark old bytes new bytes delta
BenchmarkEndToEndPipe-4 64 48 -25.00%
BenchmarkEndToEndByteBuffer-4 64 48 -25.00%
BenchmarkEndToEndSliceByteBuffer-4 34551 10554 -69.45%
BenchmarkEncodeComplex128Slice-4 55 55 +0.00%
BenchmarkEncodeFloat64Slice-4 33 33 +0.00%
BenchmarkEncodeInt32Slice-4 32 32 +0.00%
BenchmarkEncodeStringSlice-4 36 36 +0.00%
BenchmarkEncodeInterfaceSlice-4 144555 347 -99.76%
BenchmarkDecodeComplex128Slice-4 28240 28097 -0.51%
BenchmarkDecodeFloat64Slice-4 11840 11697 -1.21%
BenchmarkDecodeInt32Slice-4 10817 10673 -1.33%
BenchmarkDecodeStringSlice-4 56128 39985 -28.76%
BenchmarkDecodeInterfaceSlice-4 132565 100421 -24.25%
Change-Id: Ief7c7706b1f2916486ab7190b81aafbb16b70f1e
Reviewed-on: https://go-review.googlesource.com/13660
Reviewed-by: Russ Cox <rsc@golang.org>
diff --git a/src/encoding/gob/codec_test.go b/src/encoding/gob/codec_test.go
index 18327a6..8efcdc7 100644
--- a/src/encoding/gob/codec_test.go
+++ b/src/encoding/gob/codec_test.go
@@ -88,7 +88,6 @@
encState := newEncoderState(b)
encState.encodeInt(i)
decState := newDecodeState(newDecBuffer(b.Bytes()))
- decState.buf = make([]byte, 8)
j := decState.decodeInt()
if i != j {
t.Errorf("Encode/Decode: sent %#x received %#x", uint64(i), uint64(j))
@@ -127,7 +126,6 @@
func newDecodeState(buf *decBuffer) *decoderState {
d := new(decoderState)
d.b = buf
- d.buf = make([]byte, uint64Size)
return d
}
diff --git a/src/encoding/gob/dec_helpers.go b/src/encoding/gob/dec_helpers.go
index a1b6766..3aa038d 100644
--- a/src/encoding/gob/dec_helpers.go
+++ b/src/encoding/gob/dec_helpers.go
@@ -327,11 +327,12 @@
errorf("string data too long for buffer: %d", n)
}
// Read the data.
- data := make([]byte, n)
- if _, err := state.b.Read(data); err != nil {
- errorf("error decoding string: %s", err)
+ data := state.b.Bytes()
+ if len(data) < n {
+ errorf("invalid string length %d: exceeds input size %d", n, len(data))
}
- slice[i] = string(data)
+ slice[i] = string(data[:n])
+ state.b.Drop(n)
}
return true
}
diff --git a/src/encoding/gob/decgen.go b/src/encoding/gob/decgen.go
index da41a89..ef73f2d 100644
--- a/src/encoding/gob/decgen.go
+++ b/src/encoding/gob/decgen.go
@@ -112,11 +112,12 @@
errorf("string data too long for buffer: %d", n)
}
// Read the data.
- data := make([]byte, n)
- if _, err := state.b.Read(data); err != nil {
- errorf("error decoding string: %s", err)
+ data := state.b.Bytes()
+ if len(data) < n {
+ errorf("invalid string length %d: exceeds input size %d", n, len(data))
}
- slice[i] = string(data)`,
+ slice[i] = string(data[:n])
+ state.b.Drop(n)`,
},
{
"uint",
diff --git a/src/encoding/gob/decode.go b/src/encoding/gob/decode.go
index ef536b3..3b0dca8 100644
--- a/src/encoding/gob/decode.go
+++ b/src/encoding/gob/decode.go
@@ -29,8 +29,7 @@
// The buffer is stored with an extra indirection because it may be replaced
// if we load a type during decode (when reading an interface value).
b *decBuffer
- fieldnum int // the last field number read.
- buf []byte
+ fieldnum int // the last field number read.
next *decoderState // for free list
}
@@ -97,7 +96,6 @@
if d == nil {
d = new(decoderState)
d.dec = dec
- d.buf = make([]byte, uint64Size)
} else {
dec.freeList = d.next
}
@@ -160,15 +158,16 @@
if n > uint64Size {
error_(errBadUint)
}
- width, err := state.b.Read(state.buf[0:n])
- if err != nil {
- error_(err)
+ buf := state.b.Bytes()
+ if len(buf) < n {
+ errorf("invalid uint data length %d: exceeds input size %d", n, len(buf))
}
// Don't need to check error; it's safe to loop regardless.
// Could check that the high byte is zero but it's not worth it.
- for _, b := range state.buf[0:width] {
+ for _, b := range buf[0:n] {
x = x<<8 | uint64(b)
}
+ state.b.Drop(n)
return x
}
@@ -397,11 +396,13 @@
errorf("bad %s slice length: %d", value.Type(), n)
}
// Read the data.
- data := make([]byte, n)
- if _, err := state.b.Read(data); err != nil {
- errorf("error decoding string: %s", err)
+ data := state.b.Bytes()
+ if len(data) < n {
+ errorf("invalid string length %d: exceeds input size %d", n, len(data))
}
- value.SetString(string(data))
+ s := string(data[:n])
+ state.b.Drop(n)
+ value.SetString(s)
}
// ignoreUint8Array skips over the data for a byte slice value with no destination.
@@ -410,8 +411,11 @@
if !ok {
errorf("slice length too large")
}
- b := make([]byte, n)
- state.b.Read(b)
+ bn := state.b.Len()
+ if bn < n {
+ errorf("invalid slice length %d: exceeds input size %d", n, bn)
+ }
+ state.b.Drop(n)
}
// Execution engine
@@ -640,9 +644,9 @@
if nr > uint64(state.b.Len()) {
errorf("invalid type name length %d: exceeds input size", nr)
}
- b := make([]byte, nr)
- state.b.Read(b)
- name := string(b)
+ n := int(nr)
+ name := string(state.b.Bytes()[:n])
+ state.b.Drop(n)
// Allocate the destination interface value.
if name == "" {
// Copy the nil interface value to the target.
@@ -689,11 +693,11 @@
if !ok {
errorf("bad interface encoding: name too large for buffer")
}
- b := make([]byte, n)
- _, err := state.b.Read(b)
- if err != nil {
- error_(err)
+ bn := state.b.Len()
+ if bn < n {
+ errorf("invalid interface value length %d: exceeds input size %d", n, bn)
}
+ state.b.Drop(n)
id := dec.decodeTypeSequence(true)
if id < 0 {
error_(dec.err)
@@ -714,11 +718,13 @@
if !ok {
errorf("GobDecoder: length too large for buffer")
}
- b := make([]byte, n)
- _, err := state.b.Read(b)
- if err != nil {
- error_(err)
+ b := state.b.Bytes()
+ if len(b) < n {
+ errorf("GobDecoder: invalid data length %d: exceeds input size %d", n, len(b))
}
+ b = b[:n]
+ state.b.Drop(n)
+ var err error
// We know it's one of these.
switch ut.externalDec {
case xGob:
@@ -740,11 +746,11 @@
if !ok {
errorf("GobDecoder: length too large for buffer")
}
- b := make([]byte, n)
- _, err := state.b.Read(b)
- if err != nil {
- error_(err)
+ bn := state.b.Len()
+ if bn < n {
+ errorf("GobDecoder: invalid data length %d: exceeds input size %d", n, bn)
}
+ state.b.Drop(n)
}
// Index by Go types.
diff --git a/src/encoding/gob/encode.go b/src/encoding/gob/encode.go
index f66279f..96052ef 100644
--- a/src/encoding/gob/encode.go
+++ b/src/encoding/gob/encode.go
@@ -10,6 +10,7 @@
"encoding"
"math"
"reflect"
+ "sync"
)
const uint64Size = 8
@@ -36,6 +37,14 @@
scratch [64]byte
}
+var encBufferPool = sync.Pool{
+ New: func() interface{} {
+ e := new(encBuffer)
+ e.data = e.scratch[0:0]
+ return e
+ },
+}
+
func (e *encBuffer) WriteByte(c byte) {
e.data = append(e.data, c)
}
@@ -58,7 +67,11 @@
}
func (e *encBuffer) Reset() {
- e.data = e.data[0:0]
+ if len(e.data) >= tooBig {
+ e.data = e.scratch[0:0]
+ } else {
+ e.data = e.data[0:0]
+ }
}
func (enc *Encoder) newEncoderState(b *encBuffer) *encoderState {
@@ -407,7 +420,7 @@
// Encode the value into a new buffer. Any nested type definitions
// should be written to b, before the encoded value.
enc.pushWriter(b)
- data := new(encBuffer)
+ data := encBufferPool.Get().(*encBuffer)
data.Write(spaceForLength)
enc.encode(data, elem, ut)
if enc.err != nil {
@@ -415,6 +428,8 @@
}
enc.popWriter()
enc.writeMessage(b, data)
+ data.Reset()
+ encBufferPool.Put(data)
if enc.err != nil {
error_(enc.err)
}
diff --git a/src/encoding/gob/encoder_test.go b/src/encoding/gob/encoder_test.go
index dc65734..570d796 100644
--- a/src/encoding/gob/encoder_test.go
+++ b/src/encoding/gob/encoder_test.go
@@ -978,7 +978,7 @@
{"0f1000fb285d003316020735ff023a65c5", "interface encoding", nil},
{"03fffb0616fffc00f902ff02ff03bf005d02885802a311a8120228022c028ee7", "GobDecoder", nil},
// Issue 10491.
- {"10fe010f020102fe01100001fe010e000016fe010d030102fe010e00010101015801fe01100000000bfe011000f85555555555555555", "length exceeds input size", nil},
+ {"10fe010f020102fe01100001fe010e000016fe010d030102fe010e00010101015801fe01100000000bfe011000f85555555555555555", "exceeds input size", nil},
}
// TestBadData tests that various problems caused by malformed input
diff --git a/src/encoding/gob/timing_test.go b/src/encoding/gob/timing_test.go
index 940e5ad..424b7e6 100644
--- a/src/encoding/gob/timing_test.go
+++ b/src/encoding/gob/timing_test.go
@@ -127,8 +127,8 @@
t.Fatal("decode:", err)
}
})
- if allocs != 4 {
- t.Fatalf("mallocs per decode of type Bench: %v; wanted 4\n", allocs)
+ if allocs != 3 {
+ t.Fatalf("mallocs per decode of type Bench: %v; wanted 3\n", allocs)
}
}
@@ -200,6 +200,23 @@
}
}
+func BenchmarkEncodeInterfaceSlice(b *testing.B) {
+ var buf bytes.Buffer
+ enc := NewEncoder(&buf)
+ a := make([]interface{}, 1000)
+ for i := range a {
+ a[i] = "now is the time"
+ }
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ buf.Reset()
+ err := enc.Encode(a)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+}
+
// benchmarkBuf is a read buffer we can reset
type benchmarkBuf struct {
offset int
@@ -323,3 +340,27 @@
}
}
}
+
+func BenchmarkDecodeInterfaceSlice(b *testing.B) {
+ var buf bytes.Buffer
+ enc := NewEncoder(&buf)
+ a := make([]interface{}, 1000)
+ for i := range a {
+ a[i] = "now is the time"
+ }
+ err := enc.Encode(a)
+ if err != nil {
+ b.Fatal(err)
+ }
+ x := make([]interface{}, 1000)
+ bbuf := benchmarkBuf{data: buf.Bytes()}
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ bbuf.reset()
+ dec := NewDecoder(&bbuf)
+ err := dec.Decode(&x)
+ if err != nil {
+ b.Fatal(i, err)
+ }
+ }
+}
