Skip to content

Commit d4b89e7

Browse files
authored
Use faster gzip package (#982)
* Use faster gzip package Before: ``` BenchmarkCompression/gzip/compress-32 81 13573535 ns/op 8.77 MB/s 10115 B/op 1 allocs/op BenchmarkCompression/gzip/decompress-32 580 2023225 ns/op 959.10 MB/s 7563 B/op 53 allocs/op gzip: 93.86% ``` After: ``` BenchmarkCompression/gzip/compress-32 224 5340019 ns/op 24.42 MB/s 4838 B/op 1 allocs/op BenchmarkCompression/gzip/decompress-32 692 1713771 ns/op 1132.28 MB/s 566 B/op 34 allocs/op gzip: 93.28% ```` Pretty typical scenario for default settings, 2-3x faster, sometimes at a slight compression loss. This provides a much better "default" trade-off than the stdlib. For people regretting the 0.6% loss the higher compression levels can be used. Decompression typically has even bigger margin, but it depends on the input. * Use gz as import alias.
1 parent e6e882d commit d4b89e7

File tree

5 files changed

+10
-11
lines changed

5 files changed

+10
-11
lines changed

compress/compress_test.go

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package compress_test
22

33
import (
44
"bytes"
5-
stdgzip "compress/gzip"
65
"context"
76
"fmt"
87
"io"
@@ -16,6 +15,7 @@ import (
1615
"text/tabwriter"
1716
"time"
1817

18+
gz "github.com/klauspost/compress/gzip"
1919
"github.com/segmentio/kafka-go"
2020
pkg "github.com/segmentio/kafka-go/compress"
2121
"github.com/segmentio/kafka-go/compress/gzip"
@@ -345,7 +345,7 @@ func BenchmarkCompression(b *testing.B) {
345345
}
346346
defer f.Close()
347347

348-
z, err := stdgzip.NewReader(f)
348+
z, err := gz.NewReader(f)
349349
if err != nil {
350350
b.Fatal(err)
351351
}
@@ -366,8 +366,6 @@ func BenchmarkCompression(b *testing.B) {
366366
fmt.Println(ts)
367367
}()
368368

369-
b.ResetTimer()
370-
371369
for i := range benchmarks {
372370
benchmark := &benchmarks[i]
373371
ratio := 0.0
@@ -389,6 +387,7 @@ func benchmarkCompression(b *testing.B, codec pkg.Codec, buf *bytes.Buffer, payl
389387
b.Run("compress", func(b *testing.B) {
390388
compressed = true
391389
r := bytes.NewReader(payload)
390+
b.ReportAllocs()
392391

393392
for i := 0; i < b.N; i++ {
394393
buf.Reset()
@@ -422,7 +421,7 @@ func benchmarkCompression(b *testing.B, codec pkg.Codec, buf *bytes.Buffer, payl
422421

423422
b.Run("decompress", func(b *testing.B) {
424423
c := bytes.NewReader(buf.Bytes())
425-
424+
b.ReportAllocs()
426425
for i := 0; i < b.N; i++ {
427426
c.Reset(buf.Bytes())
428427
r := codec.NewReader(c)

compress/gzip/gzip.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
package gzip
22

33
import (
4-
"compress/gzip"
54
"io"
65
"sync"
6+
7+
"github.com/klauspost/compress/gzip"
78
)
89

910
var (

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ module github.com/segmentio/kafka-go
33
go 1.15
44

55
require (
6-
github.com/klauspost/compress v1.15.7
6+
github.com/klauspost/compress v1.15.9
77
github.com/pierrec/lz4/v4 v4.1.15
88
github.com/stretchr/testify v1.8.0
99
github.com/xdg/scram v1.0.5

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
22
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
33
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
4-
github.com/klauspost/compress v1.15.7 h1:7cgTQxJCU/vy+oP/E3B9RGbQTgbiVzIJWIKOLoAsPok=
5-
github.com/klauspost/compress v1.15.7/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
4+
github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
5+
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
66
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
77
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
88
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

gzip/gzip.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,7 @@
44
package gzip
55

66
import (
7-
gz "compress/gzip"
8-
7+
gz "github.com/klauspost/compress/gzip"
98
"github.com/segmentio/kafka-go/compress/gzip"
109
)
1110

0 commit comments

Comments
 (0)