Skip to content

Commit 0e141ce

Browse files
wenfengwangwenfengwang
authored
feat: add controller proxy for vsctl communication (#117)
* feat: add proxy for the controller in gateway
1 parent a0563de commit 0e141ce

File tree

23 files changed

+1998
-214
lines changed

23 files changed

+1998
-214
lines changed

‎cmd/gateway/main.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,14 @@ func main() {
4141
go gateway.MustStartHTTP(*cfg)
4242

4343
ga := gateway.NewGateway(*cfg)
44+
err = ga.StartCtrlProxy(context.Background())
45+
if err != nil {
46+
log.Error(context.Background(), "start controller proxy failed", map[string]interface{}{
47+
log.KeyError: err,
48+
})
49+
os.Exit(-1)
50+
}
51+
4452
err = ga.StartReceive(context.Background())
4553
if err != nil {
4654
log.Error(context.Background(), "start CloudEvents gateway failed", map[string]interface{}{

‎deploy/all-in-one.yaml‎

Lines changed: 9 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ data:
99
name: ${POD_NAME}
1010
ip: ${POD_IP}
1111
port: 2048
12-
gateway_endpoint: http://192.168.49.2:30001
1312
etcd:
1413
- vanus-controller-0.vanus-controller.vanus.svc:2379
1514
- vanus-controller-1.vanus-controller.vanus.svc:2379
@@ -96,48 +95,6 @@ spec:
9695
---
9796
apiVersion: v1
9897
kind: Service
99-
metadata:
100-
name: vanus-controller-0
101-
namespace: vanus
102-
spec:
103-
ports:
104-
- name: grpc
105-
nodePort: 32000
106-
port: 2048
107-
selector:
108-
statefulset.kubernetes.io/pod-name: vanus-controller-0
109-
type: NodePort
110-
---
111-
apiVersion: v1
112-
kind: Service
113-
metadata:
114-
name: vanus-controller-1
115-
namespace: vanus
116-
spec:
117-
ports:
118-
- name: grpc
119-
nodePort: 32001
120-
port: 2048
121-
selector:
122-
statefulset.kubernetes.io/pod-name: vanus-controller-1
123-
type: NodePort
124-
---
125-
apiVersion: v1
126-
kind: Service
127-
metadata:
128-
name: vanus-controller-2
129-
namespace: vanus
130-
spec:
131-
ports:
132-
- name: grpc
133-
nodePort: 32002
134-
port: 2048
135-
selector:
136-
statefulset.kubernetes.io/pod-name: vanus-controller-2
137-
type: NodePort
138-
---
139-
apiVersion: v1
140-
kind: Service
14198
metadata:
14299
name: vanus-gateway
143100
namespace: vanus
@@ -151,6 +108,10 @@ spec:
151108
nodePort: 30002
152109
port: 8081
153110
targetPort: 8081
111+
- name: ctrl-proxy
112+
nodePort: 30003
113+
port: 8082
114+
targetPort: 8082
154115
selector:
155116
app: vanus-gateway
156117
type: NodePort
@@ -173,7 +134,7 @@ spec:
173134
app: vanus-gateway
174135
spec:
175136
containers:
176-
- image: public.ecr.aws/vanus/gateway:v0.0.5
137+
- image: public.ecr.aws/vanus/gateway:v0.1.0
177138
imagePullPolicy: IfNotPresent
178139
name: gateway
179140
ports:
@@ -214,7 +175,7 @@ spec:
214175
valueFrom:
215176
fieldRef:
216177
fieldPath: status.podIP
217-
image: public.ecr.aws/vanus/trigger:v0.0.5
178+
image: public.ecr.aws/vanus/trigger:v0.1.0
218179
imagePullPolicy: IfNotPresent
219180
livenessProbe:
220181
failureThreshold: 3
@@ -264,7 +225,7 @@ spec:
264225
valueFrom:
265226
fieldRef:
266227
fieldPath: status.podIP
267-
image: public.ecr.aws/vanus/controller:v0.0.5
228+
image: public.ecr.aws/vanus/controller:v0.1.0
268229
imagePullPolicy: IfNotPresent
269230
livenessProbe:
270231
failureThreshold: 3
@@ -322,15 +283,15 @@ spec:
322283
- command:
323284
- /bin/sh
324285
- -c
325-
- export VOLUME_ID=${HOSTNAME##*-}; /vanus/bin/store
286+
- VOLUME_ID=${HOSTNAME##*-} /vanus/bin/store
326287
env:
327288
- name: VANUS_LOG_LEVEL
328289
value: DEBUG
329290
- name: POD_IP
330291
valueFrom:
331292
fieldRef:
332293
fieldPath: status.podIP
333-
image: public.ecr.aws/vanus/store:v0.0.5
294+
image: public.ecr.aws/vanus/store:v0.1.0
334295
imagePullPolicy: IfNotPresent
335296
livenessProbe:
336297
failureThreshold: 3

‎deploy/yaml/controller.yaml‎

Lines changed: 1 addition & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ data:
2121
name: ${POD_NAME}
2222
ip: ${POD_IP}
2323
port: 2048
24-
gateway_endpoint: http://192.168.49.2:30001
2524
etcd:
2625
- vanus-controller-0.vanus-controller.vanus.svc:2379
2726
- vanus-controller-1.vanus-controller.vanus.svc:2379
@@ -57,48 +56,6 @@ spec:
5756
- port: 2048
5857
name: vanus-controller
5958
---
60-
apiVersion: v1
61-
kind: Service
62-
metadata:
63-
name: vanus-controller-0
64-
namespace: vanus
65-
spec:
66-
selector:
67-
statefulset.kubernetes.io/pod-name: vanus-controller-0
68-
type: NodePort
69-
ports:
70-
- port: 2048
71-
nodePort: 32000
72-
name: grpc
73-
---
74-
apiVersion: v1
75-
kind: Service
76-
metadata:
77-
name: vanus-controller-1
78-
namespace: vanus
79-
spec:
80-
selector:
81-
statefulset.kubernetes.io/pod-name: vanus-controller-1
82-
type: NodePort
83-
ports:
84-
- port: 2048
85-
nodePort: 32001
86-
name: grpc
87-
---
88-
apiVersion: v1
89-
kind: Service
90-
metadata:
91-
name: vanus-controller-2
92-
namespace: vanus
93-
spec:
94-
selector:
95-
statefulset.kubernetes.io/pod-name: vanus-controller-2
96-
type: NodePort
97-
ports:
98-
- port: 2048
99-
nodePort: 32002
100-
name: grpc
101-
---
10259
apiVersion: apps/v1
10360
kind: StatefulSet
10461
metadata:
@@ -130,7 +87,7 @@ spec:
13087
spec:
13188
containers:
13289
- name: controller
133-
image: public.ecr.aws/vanus/controller:v0.0.5
90+
image: public.ecr.aws/vanus/controller:v0.1.0
13491
imagePullPolicy: IfNotPresent
13592
livenessProbe:
13693
grpc:

‎deploy/yaml/gateway.yaml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,10 @@ spec:
2929
targetPort: 8081
3030
nodePort: 30002
3131
name: get
32-
32+
- name: ctrl-proxy
33+
nodePort: 30003
34+
port: 8082
35+
targetPort: 8082
3336
---
3437
apiVersion: v1
3538
kind: ConfigMap
@@ -64,13 +67,15 @@ spec:
6467
spec:
6568
containers:
6669
- name: gateway
67-
image: public.ecr.aws/vanus/gateway:v0.0.5
70+
image: public.ecr.aws/vanus/gateway:v0.1.0
6871
imagePullPolicy: IfNotPresent
6972
ports:
7073
- name: httpput
7174
containerPort: 8080
7275
- name: httpget
7376
containerPort: 8081
77+
- name: ctrl-proxy
78+
containerPort: 8082
7479
volumeMounts:
7580
- name: config-gateway
7681
mountPath: /vanus/config

‎deploy/yaml/store.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ spec:
6161
spec:
6262
containers:
6363
- name: store
64-
image: public.ecr.aws/vanus/store:v0.0.5
64+
image: public.ecr.aws/vanus/store:v0.1.0
6565
imagePullPolicy: IfNotPresent
6666
command: ["/bin/sh", "-c", "VOLUME_ID=${HOSTNAME##*-} /vanus/bin/store"]
6767
livenessProbe:

‎deploy/yaml/trigger.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ spec:
4545
spec:
4646
containers:
4747
- name: trigger
48-
image: public.ecr.aws/vanus/trigger:v0.0.5
48+
image: public.ecr.aws/vanus/trigger:v0.1.0
4949
imagePullPolicy: IfNotPresent
5050
livenessProbe:
5151
grpc:

‎deploy/yaml/vsctl.yaml‎

Lines changed: 0 additions & 22 deletions
This file was deleted.

‎go.mod‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ require (
2222
github.com/linkall-labs/vanus/client v0.1.0
2323
github.com/linkall-labs/vanus/proto v0.1.0
2424
github.com/linkall-labs/vanus/raft v0.1.0
25+
github.com/mwitkow/grpc-proxy v0.0.0
2526
github.com/ncw/directio v1.0.5
2627
github.com/pkg/errors v0.9.1
2728
github.com/prometheus/client_golang v1.11.1
@@ -45,6 +46,7 @@ replace (
4546
github.com/linkall-labs/vanus/client => ./client
4647
github.com/linkall-labs/vanus/proto => ./proto
4748
github.com/linkall-labs/vanus/raft => ./raft
49+
github.com/mwitkow/grpc-proxy => github.com/linkall-labs/grpc-proxy v0.0.0-20220624142509-a3b0cb2bb86c
4850
)
4951

5052
require (

‎go.sum‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -338,6 +338,8 @@ github.com/labstack/gommon v0.3.1 h1:OomWaJXm7xR6L1HmEtGyQf26TEn7V6X88mktX9kee9o
338338
github.com/labstack/gommon v0.3.1/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM=
339339
github.com/linkall-labs/embed-etcd v0.0.1 h1:k1707kTZXS/HRqvt08K8fI3INJHiX1/VMPft5GWml0I=
340340
github.com/linkall-labs/embed-etcd v0.0.1/go.mod h1:2vIGENtcpLaQZ1JZvKq0LRUjvTlNCeY0r3ySTKwX0EA=
341+
github.com/linkall-labs/grpc-proxy v0.0.0-20220624142509-a3b0cb2bb86c h1:Psf/nuPqSfn3yF/AInrDmIs9bY2Rv56Cmf855i6//K0=
342+
github.com/linkall-labs/grpc-proxy v0.0.0-20220624142509-a3b0cb2bb86c/go.mod h1:MvMXoufZAtqExNexqi4cjrNYE9MefKddKylxjS+//n0=
341343
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
342344
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
343345
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -644,6 +646,7 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v
644646
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
645647
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
646648
golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
649+
golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
647650
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
648651
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
649652
golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
@@ -673,6 +676,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
673676
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
674677
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
675678
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
679+
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
676680
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
677681
golang.org/x/sys v0.0.0-20180816055513-1c9583448a9c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
678682
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -729,6 +733,7 @@ golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7w
729733
golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
730734
golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
731735
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
736+
golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
732737
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
733738
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
734739
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -895,6 +900,7 @@ google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6D
895900
google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
896901
google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
897902
google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
903+
google.golang.org/genproto v0.0.0-20210401141331-865547bb08e2/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
898904
google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
899905
google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
900906
google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
@@ -973,6 +979,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
973979
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
974980
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
975981
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
982+
honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
976983
k8s.io/api v0.23.4/go.mod h1:i77F4JfyNNrhOjZF7OwwNJS5Y1S9dpwvb9iYRYRczfI=
977984
k8s.io/apimachinery v0.23.4 h1:fhnuMd/xUL3Cjfl64j5ULKZ1/J9n8NuQEgNL+WXWfdM=
978985
k8s.io/apimachinery v0.23.4/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM=

‎internal/controller/config.go‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@ type Config struct {
3434
EtcdConfig embedetcd.Config `yaml:"embed_etcd"`
3535
Topology map[string]string `yaml:"topology"`
3636
Replicas uint `yaml:"replicas"`
37-
GatewayEndpoint string `yaml:"gateway_endpoint"`
3837
}
3938

4039
func (c *Config) GetEtcdConfig() embedetcd.Config {
@@ -51,7 +50,6 @@ func (c *Config) GetEventbusCtrlConfig() eventbus.Config {
5150
KVKeyPrefix: c.MetadataConfig.KeyPrefix,
5251
Replicas: c.Replicas,
5352
Topology: c.Topology,
54-
GatewayEndpoint: c.GatewayEndpoint,
5553
}
5654
}
5755

0 commit comments

Comments
 (0)