Skip to content

Commit 6bc47e9

Browse files
carretercarreter
committed
Add a kind example with multiple KCP nodes
1 parent 4c75cf6 commit 6bc47e9

File tree

6 files changed

+361
-0
lines changed

6 files changed

+361
-0
lines changed

‎examples/kind-multinode-kcp/README.md

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
# Use apiserver-network-proxy with KIND
2+
3+
4+
Change to the `examples/kind` folder and create a `kind` cluster with the `kind.config` file
5+
6+
```sh
7+
$ kind create cluster --config kind.config
8+
Creating cluster "kind" ...
9+
DEBUG: docker/images.go:58] Image: kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72 present locally
10+
✓ Ensuring node image (kindest/node:v1.27.3) 🖼
11+
⠎⠁ Preparing nodes 📦 📦 📦
12+
13+
This node has joined the cluster:
14+
* Certificate signing request was sent to apiserver and a response was received.
15+
* The Kubelet was informed of the new secure connection details.
16+
17+
Run 'kubectl get nodes' on the control-plane to see this node join the cluster.
18+
✓ Joining worker nodes 🚜
19+
Set kubectl context to "kind-kind"
20+
You can now use your cluster with:
21+
22+
kubectl cluster-info --context kind-kind
23+
24+
Have a nice day! 👋
25+
```
26+
27+
Once the cluster is ready install the `apiserver-network-proxy` components:
28+
29+
```sh
30+
$ kubectl apply -f konnectivity-server.yaml
31+
clusterrolebinding.rbac.authorization.k8s.io/system:konnectivity-server created
32+
daemonset.apps/konnectivity-server created
33+
34+
$ kubectl apply -f konnectivity-agent-ds.yaml
35+
serviceaccount/konnectivity-agent created
36+
```
37+
38+
To validate that it works, run a custom image and get pod logs (it goes through the konnectivity proxy):
39+
```sh
40+
$ kubectl run test --image httpd:2
41+
pod/test created
42+
$ kubectl get pods
43+
NAME READY STATUS RESTARTS AGE
44+
test 0/1 ContainerCreating 0 4s
45+
$ kubectl get pods
46+
NAME READY STATUS RESTARTS AGE
47+
test 1/1 Running 0 6s
48+
$ kubectl logs test
49+
...
50+
[Tue Apr 09 20:58:36.756720 2024] [mpm_event:notice] [pid 1:tid 139788897408896] AH00489: Apache/2.4.59 (Unix) configured -- resuming normal operations
51+
```

‎examples/kind-multinode-kcp/egress_selector_configuration.yaml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
apiVersion: apiserver.k8s.io/v1beta1
2+
kind: EgressSelectorConfiguration
3+
egressSelections:
4+
- name: cluster
5+
connection:
6+
proxyProtocol: GRPC
7+
transport:
8+
uds:
9+
udsName: /etc/kubernetes/konnectivity-server/konnectivity-server.socket
10+
- name: master
11+
connection:
12+
proxyProtocol: Direct
13+
- name: etcd
14+
connection:
15+
proxyProtocol: Direct

‎examples/kind-multinode-kcp/kind.config

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
kind: Cluster
2+
apiVersion: kind.x-k8s.io/v1alpha4
3+
networking:
4+
ipFamily: ipv4
5+
nodes:
6+
- role: control-plane
7+
kubeadmConfigPatchesJSON6902:
8+
- kind: ClusterConfiguration
9+
patch: |
10+
- op: add
11+
path: /apiServer/certSANs/-
12+
value: konnectivity-server.kube-system.svc.cluster.local
13+
kubeadmConfigPatches:
14+
- |
15+
kind: ClusterConfiguration
16+
apiServer:
17+
extraArgs:
18+
"egress-selector-config-file": "/etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml"
19+
extraVolumes:
20+
- name: egress-selector-config-file
21+
hostPath: "/etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml"
22+
mountPath: "/etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml"
23+
readOnly: true
24+
- name: konnectivity-server
25+
hostPath: "/etc/kubernetes/konnectivity-server"
26+
mountPath: "/etc/kubernetes/konnectivity-server"
27+
readOnly: true
28+
extraMounts:
29+
- hostPath: ./egress_selector_configuration.yaml
30+
containerPath: /etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml
31+
- role: control-plane
32+
kubeadmConfigPatchesJSON6902:
33+
- kind: ClusterConfiguration
34+
patch: |
35+
- op: add
36+
path: /apiServer/certSANs/-
37+
value: konnectivity-server.kube-system.svc.cluster.local
38+
kubeadmConfigPatches:
39+
- |
40+
kind: ClusterConfiguration
41+
apiServer:
42+
extraArgs:
43+
"egress-selector-config-file": "/etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml"
44+
extraVolumes:
45+
- name: egress-selector-config-file
46+
hostPath: "/etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml"
47+
mountPath: "/etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml"
48+
readOnly: true
49+
- name: konnectivity-server
50+
hostPath: "/etc/kubernetes/konnectivity-server"
51+
mountPath: "/etc/kubernetes/konnectivity-server"
52+
readOnly: true
53+
extraMounts:
54+
- hostPath: ./egress_selector_configuration.yaml
55+
containerPath: /etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml
56+
- role: worker
57+
- role: worker

‎examples/kind-multinode-kcp/konnectivity-agent-ds.yaml

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
---
2+
apiVersion: v1
3+
kind: ServiceAccount
4+
metadata:
5+
name: konnectivity-agent
6+
namespace: kube-system
7+
labels:
8+
kubernetes.io/cluster-service: "true"
9+
---
10+
apiVersion: apps/v1
11+
kind: DaemonSet
12+
metadata:
13+
labels:
14+
k8s-app: konnectivity-agent
15+
namespace: kube-system
16+
name: konnectivity-agent
17+
spec:
18+
selector:
19+
matchLabels:
20+
k8s-app: konnectivity-agent
21+
updateStrategy:
22+
type: RollingUpdate
23+
template:
24+
metadata:
25+
labels:
26+
k8s-app: konnectivity-agent
27+
spec:
28+
priorityClassName: system-cluster-critical
29+
tolerations:
30+
- key: "CriticalAddonsOnly"
31+
operator: "Exists"
32+
- operator: "Exists"
33+
effect: "NoExecute"
34+
nodeSelector:
35+
kubernetes.io/os: linux
36+
dnsPolicy: ClusterFirstWithHostNet
37+
containers:
38+
- name: konnectivity-agent-container
39+
image: gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master
40+
resources:
41+
requests:
42+
cpu: 50m
43+
limits:
44+
memory: 30Mi
45+
command: [ "/proxy-agent"]
46+
args: [
47+
"--logtostderr=true",
48+
"--ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
49+
"--proxy-server-host=konnectivity-server.kube-system.svc.cluster.local",
50+
"--proxy-server-port=8091",
51+
"--sync-interval=5s",
52+
"--sync-interval-cap=30s",
53+
"--sync-forever",
54+
"--probe-interval=5s",
55+
"--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token",
56+
"--agent-identifiers=ipv4=$(HOST_IP)"
57+
]
58+
env:
59+
- name: POD_NAME
60+
valueFrom:
61+
fieldRef:
62+
fieldPath: metadata.name
63+
- name: POD_NAMESPACE
64+
valueFrom:
65+
fieldRef:
66+
fieldPath: metadata.namespace
67+
- name: HOST_IP
68+
valueFrom:
69+
fieldRef:
70+
fieldPath: status.hostIP
71+
livenessProbe:
72+
httpGet:
73+
scheme: HTTP
74+
port: 8093
75+
path: /healthz
76+
initialDelaySeconds: 15
77+
timeoutSeconds: 15
78+
readinessProbe:
79+
httpGet:
80+
scheme: HTTP
81+
port: 8093
82+
path: /readyz
83+
initialDelaySeconds: 15
84+
timeoutSeconds: 15
85+
volumeMounts:
86+
- mountPath: /var/run/secrets/tokens
87+
name: konnectivity-agent-token
88+
serviceAccountName: konnectivity-agent
89+
volumes:
90+
- name: konnectivity-agent-token
91+
projected:
92+
sources:
93+
- serviceAccountToken:
94+
path: konnectivity-agent-token
95+
audience: system:konnectivity-server

‎examples/kind-multinode-kcp/konnectivity-server.yaml

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: ClusterRoleBinding
3+
metadata:
4+
name: system:konnectivity-server
5+
labels:
6+
kubernetes.io/cluster-service: "true"
7+
roleRef:
8+
apiGroup: rbac.authorization.k8s.io
9+
kind: ClusterRole
10+
name: system:auth-delegator
11+
subjects:
12+
- apiGroup: rbac.authorization.k8s.io
13+
kind: User
14+
name: system:konnectivity-server
15+
---
16+
apiVersion: v1
17+
kind: Service
18+
metadata:
19+
name: konnectivity-server
20+
namespace: kube-system
21+
spec:
22+
selector:
23+
k8s-app: konnectivity-server
24+
clusterIP: None
25+
ports:
26+
- protocol: TCP
27+
port: 8091
28+
targetPort: 8091
29+
---
30+
apiVersion: apps/v1
31+
kind: DaemonSet
32+
metadata:
33+
labels:
34+
k8s-app: konnectivity-server
35+
namespace: kube-system
36+
name: konnectivity-server
37+
spec:
38+
selector:
39+
matchLabels:
40+
k8s-app: konnectivity-server
41+
updateStrategy:
42+
type: RollingUpdate
43+
template:
44+
metadata:
45+
labels:
46+
k8s-app: konnectivity-server
47+
spec:
48+
priorityClassName: system-cluster-critical
49+
tolerations:
50+
- key: "CriticalAddonsOnly"
51+
operator: "Exists"
52+
- operator: "Exists"
53+
nodeSelector:
54+
node-role.kubernetes.io/control-plane: ""
55+
hostNetwork: true
56+
containers:
57+
- name: konnectivity-server-container
58+
image: gcr.io/k8s-staging-kas-network-proxy/proxy-server:master
59+
resources:
60+
requests:
61+
cpu: 1m
62+
securityContext:
63+
allowPrivilegeEscalation: false
64+
runAsUser: 0
65+
command: [ "/proxy-server"]
66+
args: [
67+
"--log-file=/var/log/konnectivity-server.log",
68+
"--logtostderr=true",
69+
"--log-file-max-size=0",
70+
"--uds-name=/etc/kubernetes/konnectivity-server/konnectivity-server.socket",
71+
"--delete-existing-uds-file",
72+
"--cluster-cert=/etc/kubernetes/pki/apiserver.crt",
73+
"--cluster-key=/etc/kubernetes/pki/apiserver.key",
74+
"--server-port=0",
75+
"--agent-port=8091",
76+
"--health-port=8092",
77+
"--admin-port=8093",
78+
"--keepalive-time=1h",
79+
"--mode=grpc",
80+
"--agent-namespace=kube-system",
81+
"--agent-service-account=konnectivity-agent",
82+
"--kubeconfig=/etc/kubernetes/admin.conf",
83+
"--authentication-audience=system:konnectivity-server",
84+
"--server-count=2",
85+
]
86+
livenessProbe:
87+
httpGet:
88+
scheme: HTTP
89+
host: 127.0.0.1
90+
port: 8092
91+
path: /healthz
92+
initialDelaySeconds: 10
93+
timeoutSeconds: 60
94+
ports:
95+
- name: serverport
96+
containerPort: 8090
97+
hostPort: 8090
98+
- name: agentport
99+
containerPort: 8091
100+
hostPort: 8091
101+
- name: healthport
102+
containerPort: 8092
103+
hostPort: 8092
104+
- name: adminport
105+
containerPort: 8093
106+
hostPort: 8093
107+
volumeMounts:
108+
- name: varlogkonnectivityserver
109+
mountPath: /var/log/konnectivity-server.log
110+
readOnly: false
111+
- name: kubernetes
112+
mountPath: /etc/kubernetes
113+
readOnly: true
114+
- name: konnectivity-home
115+
mountPath: /etc/kubernetes/konnectivity-server
116+
volumes:
117+
- name: varlogkonnectivityserver
118+
hostPath:
119+
path: /var/log/konnectivity-server.log
120+
type: FileOrCreate
121+
- name: kubernetes
122+
hostPath:
123+
path: /etc/kubernetes
124+
- name: konnectivity-home
125+
hostPath:
126+
path: /etc/kubernetes/konnectivity-server
127+
type: DirectoryOrCreate

‎examples/kind-multinode-kcp/quickstart-kind.sh

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
CLUSTER_NAME=$1
6+
7+
echo "Creating cluster $CLUSTER_NAME..."
8+
kind create cluster --config kind.config --name $CLUSTER_NAME
9+
10+
echo "Successfully created cluster. Switching kubectl context to kind-$CLUSTER_NAME"
11+
kubectl cluster-info --context kind-$CLUSTER_NAME
12+
13+
echo "Requesting creation of konnectivity proxy servers on cluster $CLUSTER_NAME..."
14+
kubectl apply -f konnectivity-server.yaml
15+
echo "Requesting creation of konnectivity proxy agents on cluster $CLUSTER_NAME..."
16+
kubectl apply -f konnectivity-agent-ds.yaml

0 commit comments

Comments
 (0)