Merge pull request #382 from erizocosmico/feature/compress-keys

gitbase: compress keys using zlib

Author: ajnavarro

index.go

@@ -2,10 +2,13 @@ package gitbase
 
 import (
 	"bytes"
+	"compress/zlib"
 	"crypto/sha1"
 	"encoding/binary"
 	"fmt"
 	"io"
+	"io/ioutil"
+	"sync"
 
 	errors "gopkg.in/src-d/go-errors.v1"
 	"gopkg.in/src-d/go-mysql-server.v0/sql"
@@ -24,12 +27,54 @@ type Indexable interface {
 	gitBase
 }
 
+type zlibEncoder struct {
+	w   *zlib.Writer
+	mut sync.Mutex
+}
+
+func (e *zlibEncoder) encode(data []byte) ([]byte, error) {
+	e.mut.Lock()
+	defer e.mut.Unlock()
+
+	var buf bytes.Buffer
+	e.w.Reset(&buf)
+
+	if _, err := e.w.Write(data); err != nil {
+		return nil, err
+	}
+
+	if err := e.w.Close(); err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+var encoder = func() *zlibEncoder {
+	return &zlibEncoder{w: zlib.NewWriter(bytes.NewBuffer(nil))}
+}()
+
 func encodeIndexKey(k indexKey) ([]byte, error) {
-	return k.encode()
+	bs, err := k.encode()
+	if err != nil {
+		return nil, err
+	}
+
+	return encoder.encode(bs)
 }
 
 func decodeIndexKey(data []byte, k indexKey) error {
-	return k.decode(data)
+	gz, err := zlib.NewReader(bytes.NewReader(data))
+	if err != nil {
+		return err
+	}
+
+	bs, err := ioutil.ReadAll(gz)
+	if err != nil {
+		return err
+	}
+
+	return k.decode(bs)
 }
 
 func rowIndexValues(row sql.Row, columns []string, schema sql.Schema) ([]interface{}, error) {

index_test.go

@@ -2,7 +2,6 @@ package gitbase
 
 import (
 	"bytes"
-	"encoding/gob"
 	"io"
 	"testing"
 
@@ -121,37 +120,29 @@ func testTableIndex(
 	require.ElementsMatch(expected, rows)
 }
 
-func TestEncodeKeySize(t *testing.T) {
+func TestEncodeRoundtrip(t *testing.T) {
 	require := require.New(t)
 
-	k1 := packOffsetIndexKey{
+	k := &packOffsetIndexKey{
 		Repository: "/foo/bar/baz/repo.git",
 		Packfile:   plumbing.ZeroHash.String(),
 		Offset:     12345,
 		Hash:       "",
 	}
 
-	k2 := packOffsetIndexKey{
-		Repository: "/foo/bar/baz/repo.git",
-		Packfile:   plumbing.ZeroHash.String(),
-		Offset:     -1,
-		Hash:       plumbing.ZeroHash.String(),
-	}
-
-	var buf1, buf2 bytes.Buffer
-	require.NoError(gob.NewEncoder(&buf1).Encode(k1))
-
-	bytes1, err := k1.encode()
+	bs, err := encodeIndexKey(k)
 	require.NoError(err)
 
-	require.True(len(bytes1) < len(buf1.Bytes()))
+	bsraw, err := k.encode()
+	require.NoError(err)
 
-	require.NoError(gob.NewEncoder(&buf2).Encode(k2))
+	// check encodeIndexKey also compresses the encoded value
+	require.True(len(bs) < len(bsraw))
 
-	bytes2, err := k2.encode()
-	require.NoError(err)
+	var k2 packOffsetIndexKey
+	require.NoError(decodeIndexKey(bs, &k2))
 
-	require.True(len(bytes2) < len(buf2.Bytes()))
+	require.Equal(k, &k2)
 }
 
 func TestWriteReadInt64(t *testing.T) {