Skip to content

Commit 77eeaea

Browse files
authored
Merge pull request #100 from grepplabs/kafka-3.0
Kafka 3.0 / 3.1
2 parents 69f439e + 5e44763 commit 77eeaea

File tree

36 files changed

+138
-1545
lines changed

36 files changed

+138
-1545
lines changed

.travis.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ language: go
33
dist: focal
44

55
go:
6-
- "1.16.x"
6+
- "1.17.x"
77

88
env:
99
global:

Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM golang:1.16-alpine3.14 as builder
1+
FROM golang:1.17-alpine3.14 as builder
22
RUN apk add alpine-sdk ca-certificates
33

44
WORKDIR /go/src/github.com/grepplabs/kafka-proxy

Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ VERSION ?= $(shell git describe --tags --always --dirty)
1010
GOPKGS = $(shell go list ./... | grep -v /vendor/)
1111
BUILD_FLAGS ?=
1212
LDFLAGS ?= -X github.com/grepplabs/kafka-proxy/config.Version=$(VERSION) -w -s
13-
TAG ?= "v0.3.0"
13+
TAG ?= "v0.3.1"
1414
GOARCH ?= amd64
1515
GOOS ?= linux
1616

README.md

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ As not every Kafka release adds new messages/versions which are relevant to the
3737
| Kafka proxy version | Kafka version |
3838
| ------------------ | ----------- |
3939
| 0.2.9 | 2.8.0 |
40+
| 0.3.1 | 3.1.0 |
4041

4142

4243
### Install binary release
@@ -45,11 +46,11 @@ As not every Kafka release adds new messages/versions which are relevant to the
4546

4647
Linux
4748

48-
curl -Ls https://github.com/grepplabs/kafka-proxy/releases/download/v0.3.0/kafka-proxy-v0.3.0-linux-amd64.tar.gz | tar xz
49+
curl -Ls https://github.com/grepplabs/kafka-proxy/releases/download/v0.3.1/kafka-proxy-v0.3.1-linux-amd64.tar.gz | tar xz
4950

5051
macOS
5152

52-
curl -Ls https://github.com/grepplabs/kafka-proxy/releases/download/v0.3.0/kafka-proxy-v0.3.0-darwin-amd64.tar.gz | tar xz
53+
curl -Ls https://github.com/grepplabs/kafka-proxy/releases/download/v0.3.1/kafka-proxy-v0.3.1-darwin-amd64.tar.gz | tar xz
5354

5455
2. Move the binary in to your PATH.
5556

@@ -67,7 +68,7 @@ Docker images are available on [Docker Hub](https://hub.docker.com/r/grepplabs/k
6768
6869
You can launch a kafka-proxy container for trying it out with
6970
70-
docker run --rm -p 30001-30003:30001-30003 grepplabs/kafka-proxy:v0.3.0 \
71+
docker run --rm -p 30001-30003:30001-30003 grepplabs/kafka-proxy:v0.3.1 \
7172
server \
7273
--bootstrap-server-mapping "localhost:19092,0.0.0.0:30001" \
7374
--bootstrap-server-mapping "localhost:29092,0.0.0.0:30002" \
@@ -86,7 +87,7 @@ Docker images with precompiled plugins located in `/opt/kafka-proxy/bin/` are ta
8687
8788
You can launch a kafka-proxy container with auth-ldap plugin for trying it out with
8889
89-
docker run --rm -p 30001-30003:30001-30003 grepplabs/kafka-proxy:v0.3.0-all \
90+
docker run --rm -p 30001-30003:30001-30003 grepplabs/kafka-proxy:v0.3.1-all \
9091
server \
9192
--bootstrap-server-mapping "localhost:19092,0.0.0.0:30001" \
9293
--bootstrap-server-mapping "localhost:29092,0.0.0.0:30002" \

go.mod

Lines changed: 32 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,65 @@
11
module github.com/grepplabs/kafka-proxy
22

3-
go 1.16
3+
go 1.17
44

55
require (
6-
cloud.google.com/go v0.19.0 // indirect
7-
github.com/BurntSushi/toml v0.3.1 // indirect
86
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
97
github.com/cenkalti/backoff v1.1.0
108
github.com/elazarl/goproxy v0.0.0-20171101143503-a96fa3a31826
119
github.com/fsnotify/fsnotify v1.4.9
1210
github.com/go-ldap/ldap/v3 v3.2.3
13-
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
1411
github.com/golang/protobuf v1.4.2
1512
github.com/google/uuid v1.2.0
16-
github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect
1713
github.com/hashicorp/go-hclog v0.0.0-20180122232401-5bcb0f17e364
1814
github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874
1915
github.com/hashicorp/go-plugin v0.0.0-20180314222826-8068b0bdcfb7
16+
github.com/oklog/run v1.1.0
17+
github.com/pkg/errors v0.9.1
18+
github.com/prometheus/client_golang v1.7.1
19+
github.com/sirupsen/logrus v1.6.0
20+
github.com/spf13/cobra v0.0.1
21+
github.com/spf13/viper v1.0.2
22+
github.com/stretchr/testify v1.4.0
23+
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
24+
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7
25+
golang.org/x/oauth2 v0.0.0-20180314180239-fdc9e635145a
26+
google.golang.org/api v0.0.0-20180313183023-c24aa0e5ed34
27+
google.golang.org/grpc v1.10.0
28+
)
29+
30+
require (
31+
cloud.google.com/go v0.19.0 // indirect
32+
github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c // indirect
33+
github.com/BurntSushi/toml v0.3.1 // indirect
34+
github.com/beorn7/perks v1.0.1 // indirect
35+
github.com/cespare/xxhash/v2 v2.1.1 // indirect
36+
github.com/davecgh/go-spew v1.1.1 // indirect
37+
github.com/go-asn1-ber/asn1-ber v1.5.1 // indirect
38+
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
39+
github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect
2040
github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce // indirect
2141
github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d // indirect
2242
github.com/inconshreveable/mousetrap v1.0.0 // indirect
23-
github.com/klauspost/cpuid v1.2.0
43+
github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
2444
github.com/magiconair/properties v1.8.0 // indirect
45+
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
2546
github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 // indirect
2647
github.com/mitchellh/mapstructure v0.0.0-20180511142126-bb74f1db0675 // indirect
27-
github.com/oklog/run v1.1.0
2848
github.com/pelletier/go-toml v1.2.0 // indirect
29-
github.com/pkg/errors v0.9.1
30-
github.com/prometheus/client_golang v1.7.1
31-
github.com/sirupsen/logrus v1.6.0
49+
github.com/pmezard/go-difflib v1.0.0 // indirect
50+
github.com/prometheus/client_model v0.2.0 // indirect
51+
github.com/prometheus/common v0.10.0 // indirect
52+
github.com/prometheus/procfs v0.1.3 // indirect
3253
github.com/spf13/afero v1.1.1 // indirect
3354
github.com/spf13/cast v1.2.0 // indirect
34-
github.com/spf13/cobra v0.0.1
3555
github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec // indirect
3656
github.com/spf13/pflag v1.0.0 // indirect
37-
github.com/spf13/viper v1.0.2
38-
github.com/stretchr/testify v1.4.0
39-
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
4057
github.com/xdg/stringprep v1.0.0 // indirect
4158
golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f // indirect
42-
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7
43-
golang.org/x/oauth2 v0.0.0-20180314180239-fdc9e635145a
4459
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect
4560
golang.org/x/text v0.3.2 // indirect
46-
google.golang.org/api v0.0.0-20180313183023-c24aa0e5ed34
4761
google.golang.org/appengine v1.0.0 // indirect
4862
google.golang.org/genproto v0.0.0-20180316064809-f8c870359523 // indirect
49-
google.golang.org/grpc v1.10.0
63+
google.golang.org/protobuf v1.23.0 // indirect
5064
gopkg.in/yaml.v2 v2.3.0 // indirect
5165
)

go.sum

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -71,8 +71,6 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
7171
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
7272
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
7373
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
74-
github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE=
75-
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
7674
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
7775
github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
7876
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -149,8 +147,8 @@ github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0
149147
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
150148
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
151149
golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
152-
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM=
153-
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
150+
golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f h1:aZp0e2vLN4MToVqnjNEYEtrEA8RH8U8FN1CU7JgqsPU=
151+
golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
154152
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
155153
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
156154
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -167,11 +165,13 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
167165
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
168166
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
169167
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
168+
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
170169
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
171170
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
172171
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
173172
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo=
174173
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
174+
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
175175
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
176176
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
177177
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=

proxy/protocol/responses.go

Lines changed: 55 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ const (
1515
hostKeyName = "host"
1616
portKeyName = "port"
1717

18-
coordinatorKeyName = "coordinator"
18+
coordinatorKeyName = "coordinator"
19+
coordinatorsKeyName = "coordinators"
1920
)
2021

2122
var (
@@ -144,6 +145,16 @@ func createMetadataResponseSchemaVersions() []Schema {
144145
&SchemaTaggedFields{Name: "topic_metadata_tagged_fields"},
145146
)
146147

148+
topicMetadataSchema12 := NewSchema("topic_metadata_schema12",
149+
&Mfield{Name: "error_code", Ty: TypeInt16},
150+
&Mfield{Name: "name", Ty: TypeCompactNullableStr},
151+
&Mfield{Name: "topic_id", Ty: TypeUuid},
152+
&Mfield{Name: "is_internal", Ty: TypeBool},
153+
&CompactArray{Name: "partition_metadata", Ty: partitionMetadataSchema9},
154+
&Mfield{Name: "topic_authorized_operations", Ty: TypeInt32},
155+
&SchemaTaggedFields{Name: "topic_metadata_tagged_fields"},
156+
)
157+
147158
metadataResponseV1 := NewSchema("metadata_response_v1",
148159
&Array{Name: brokersKeyName, Ty: metadataBrokerV1},
149160
&Mfield{Name: "controller_id", Ty: TypeInt32},
@@ -222,7 +233,17 @@ func createMetadataResponseSchemaVersions() []Schema {
222233
&CompactArray{Name: "topic_metadata", Ty: topicMetadataSchema10},
223234
&SchemaTaggedFields{Name: "response_tagged_fields"},
224235
)
225-
return []Schema{metadataResponseV0, metadataResponseV1, metadataResponseV2, metadataResponseV3, metadataResponseV4, metadataResponseV5, metadataResponseV6, metadataResponseV7, metadataResponseV8, metadataResponseV9, metadataResponseV10, metadataResponseV11}
236+
237+
metadataResponseV12 := NewSchema("metadata_response_v12",
238+
&Mfield{Name: "throttle_time_ms", Ty: TypeInt32},
239+
&CompactArray{Name: brokersKeyName, Ty: metadataBrokerSchema9},
240+
&Mfield{Name: "cluster_id", Ty: TypeCompactNullableStr},
241+
&Mfield{Name: "controller_id", Ty: TypeInt32},
242+
&CompactArray{Name: "topic_metadata", Ty: topicMetadataSchema12},
243+
&SchemaTaggedFields{Name: "response_tagged_fields"},
244+
)
245+
246+
return []Schema{metadataResponseV0, metadataResponseV1, metadataResponseV2, metadataResponseV3, metadataResponseV4, metadataResponseV5, metadataResponseV6, metadataResponseV7, metadataResponseV8, metadataResponseV9, metadataResponseV10, metadataResponseV11, metadataResponseV12}
226247
}
227248

228249
func createFindCoordinatorResponseSchemaVersions() []Schema {
@@ -260,7 +281,19 @@ func createFindCoordinatorResponseSchemaVersions() []Schema {
260281
&SchemaTaggedFields{Name: "response_tagged_fields"},
261282
)
262283

263-
return []Schema{findCoordinatorResponseV0, findCoordinatorResponseV1, findCoordinatorResponseV2, findCoordinatorResponseV3}
284+
findCoordinatorCoordinatorsSchema4 := NewSchema("find_coordinator_coordinators_schema4",
285+
&Mfield{Name: "key", Ty: TypeCompactStr},
286+
&Mfield{Name: coordinatorKeyName, Ty: findCoordinatorBrokerSchema9},
287+
&Mfield{Name: "error_code", Ty: TypeInt16},
288+
&Mfield{Name: "error_message", Ty: TypeCompactNullableStr},
289+
&SchemaTaggedFields{"coordinators_tagged_fields"},
290+
)
291+
findCoordinatorResponseV4 := NewSchema("find_coordinator_response_v4",
292+
&Mfield{Name: "throttle_time_ms", Ty: TypeInt32},
293+
&CompactArray{Name: coordinatorsKeyName, Ty: findCoordinatorCoordinatorsSchema4},
294+
&SchemaTaggedFields{Name: "response_tagged_fields"},
295+
)
296+
return []Schema{findCoordinatorResponseV0, findCoordinatorResponseV1, findCoordinatorResponseV2, findCoordinatorResponseV3, findCoordinatorResponseV4}
264297
}
265298

266299
func modifyMetadataResponse(decodedStruct *Struct, fn config.NetAddressMappingFunc) error {
@@ -316,6 +349,25 @@ func modifyFindCoordinatorResponse(decodedStruct *Struct, fn config.NetAddressMa
316349
if fn == nil {
317350
return errors.New("net address mapper must not be nil")
318351
}
352+
coordinators := decodedStruct.Get(coordinatorsKeyName)
353+
if coordinators != nil {
354+
coordinatorsArray, ok := coordinators.([]interface{})
355+
if !ok {
356+
return errors.New("coordinators list not found")
357+
}
358+
for _, coordinatorElement := range coordinatorsArray {
359+
coordinatorStruct := coordinatorElement.(*Struct)
360+
if err := modifyCoordinator(coordinatorStruct, fn); err != nil {
361+
return err
362+
}
363+
}
364+
return nil
365+
} else {
366+
return modifyCoordinator(decodedStruct, fn)
367+
}
368+
}
369+
370+
func modifyCoordinator(decodedStruct *Struct, fn config.NetAddressMappingFunc) error {
319371
coordinator, ok := decodedStruct.Get(coordinatorKeyName).(*Struct)
320372
if !ok {
321373
return errors.New("coordinator not found")

0 commit comments

Comments
 (0)