Skip to content
This repository was archived by the owner on Jul 30, 2021. It is now read-only.

Commit 2843058

Browse files
author
Abhinav Dahiya
committed
Add support for network policy (#513)
Notes: - mounts empty /opt/cni/bin from host to kubelet container(pod) so that all pods share same cni binaries Signed-off-by: Abhinav Dahiya <[email protected]>
1 parent 03d6038 commit 2843058

File tree

13 files changed

+611
-29
lines changed

13 files changed

+611
-29
lines changed

‎cmd/bootkube/render.go‎

Lines changed: 24 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,11 @@ var (
5252
selfHostKubelet bool
5353
cloudProvider string
5454
selfHostedEtcd bool
55+
calicoNetworkPolicy bool
5556
}
5657

5758
imageVersions = asset.DefaultImages
59+
cniRelease = asset.DefaultCNIRelease
5860
)
5961

6062
func init() {
@@ -73,6 +75,7 @@ func init() {
7375
cmdRender.Flags().BoolVar(&renderOpts.selfHostKubelet, "experimental-self-hosted-kubelet", false, "(Experimental) Create a self-hosted kubelet daemonset.")
7476
cmdRender.Flags().StringVar(&renderOpts.cloudProvider, "cloud-provider", "", "The provider for cloud services. Empty string for no provider")
7577
cmdRender.Flags().BoolVar(&renderOpts.selfHostedEtcd, "experimental-self-hosted-etcd", false, "(Experimental) Create self-hosted etcd assets.")
78+
cmdRender.Flags().BoolVar(&renderOpts.calicoNetworkPolicy, "experimental-calico-network-policy", false, "(Experimental) Add network policy support by calico.")
7679
}
7780

7881
func runCmdRender(cmd *cobra.Command, args []string) error {
@@ -222,25 +225,27 @@ func flagsToAssetConfig() (c *asset.Config, err error) {
222225
}
223226

224227
return &asset.Config{
225-
EtcdCACert: etcdCACert,
226-
EtcdClientCert: etcdClientCert,
227-
EtcdClientKey: etcdClientKey,
228-
EtcdServers: etcdServers,
229-
EtcdUseTLS: etcdUseTLS,
230-
CACert: caCert,
231-
CAPrivKey: caPrivKey,
232-
APIServers: apiServers,
233-
AltNames: altNames,
234-
PodCIDR: podNet,
235-
ServiceCIDR: serviceNet,
236-
APIServiceIP: apiServiceIP,
237-
BootEtcdServiceIP: bootEtcdServiceIP,
238-
DNSServiceIP: dnsServiceIP,
239-
EtcdServiceIP: etcdServiceIP,
240-
SelfHostKubelet: renderOpts.selfHostKubelet,
241-
CloudProvider: renderOpts.cloudProvider,
242-
SelfHostedEtcd: renderOpts.selfHostedEtcd,
243-
Images: imageVersions,
228+
EtcdCACert: etcdCACert,
229+
EtcdClientCert: etcdClientCert,
230+
EtcdClientKey: etcdClientKey,
231+
EtcdServers: etcdServers,
232+
EtcdUseTLS: etcdUseTLS,
233+
CACert: caCert,
234+
CAPrivKey: caPrivKey,
235+
APIServers: apiServers,
236+
AltNames: altNames,
237+
PodCIDR: podNet,
238+
ServiceCIDR: serviceNet,
239+
APIServiceIP: apiServiceIP,
240+
BootEtcdServiceIP: bootEtcdServiceIP,
241+
DNSServiceIP: dnsServiceIP,
242+
EtcdServiceIP: etcdServiceIP,
243+
SelfHostKubelet: renderOpts.selfHostKubelet,
244+
CloudProvider: renderOpts.cloudProvider,
245+
SelfHostedEtcd: renderOpts.selfHostedEtcd,
246+
CalicoNetworkPolicy: renderOpts.calicoNetworkPolicy,
247+
Images: imageVersions,
248+
CNIRelease: cniRelease,
244249
}, nil
245250
}
246251

‎e2e/network_test.go‎

Lines changed: 295 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,295 @@
1+
package e2e
2+
3+
import (
4+
"fmt"
5+
"strings"
6+
"testing"
7+
"time"
8+
9+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
10+
utilrand "k8s.io/apimachinery/pkg/util/rand"
11+
"k8s.io/client-go/pkg/api"
12+
"k8s.io/client-go/pkg/api/v1"
13+
"k8s.io/client-go/pkg/apis/extensions/v1beta1"
14+
)
15+
16+
func TestNetwork(t *testing.T) {
17+
//
18+
// 1. create nginx service
19+
di, _, err := api.Codecs.UniversalDecoder().Decode(nginxDepNT, nil, &v1beta1.Deployment{})
20+
if err != nil {
21+
t.Fatalf("unable to decode deployment manifest: %v", err)
22+
}
23+
24+
d, ok := di.(*v1beta1.Deployment)
25+
if !ok {
26+
t.Fatalf("expected manifest to decode into *api.deployment, got %T", di)
27+
}
28+
_, err = client.ExtensionsV1beta1().Deployments(namespace).Create(d)
29+
if err != nil {
30+
t.Fatal(err)
31+
}
32+
33+
deleteDeployment := func() {
34+
delPropPolicy := metav1.DeletePropagationForeground
35+
client.ExtensionsV1beta1().Deployments(namespace).Delete("nginx-deployment-nt", &metav1.DeleteOptions{
36+
PropagationPolicy: &delPropPolicy,
37+
})
38+
}
39+
defer deleteDeployment()
40+
41+
if err := retry(10, time.Second*10, getNginxPod); err != nil {
42+
t.Fatalf("timed out waiting for nginx pod: %v", err)
43+
}
44+
45+
si, _, err := api.Codecs.UniversalDecoder().Decode(nginxSVCNT, nil, &v1.Service{})
46+
if err != nil {
47+
t.Fatalf("unable to decode service manifest: %v", err)
48+
}
49+
s, ok := si.(*v1.Service)
50+
if !ok {
51+
t.Fatalf("expected manifest to decode into *api.service, got %T", si)
52+
}
53+
_, err = client.CoreV1().Services(namespace).Create(s)
54+
if err != nil {
55+
t.Fatal(err)
56+
}
57+
defer client.CoreV1().Services(namespace).Delete("nginx-service-nt", &metav1.DeleteOptions{})
58+
59+
//
60+
// 2. create a wget pod that hits the nginx service
61+
testPodName := fmt.Sprintf("%s-%s", "wget-pod-nt", utilrand.String(5))
62+
wgetPodNT.ObjectMeta.Name = testPodName
63+
_, err = client.CoreV1().Pods(namespace).Create(wgetPodNT)
64+
if err != nil {
65+
t.Fatal(err)
66+
}
67+
68+
if err := retry(10, time.Second*10, getPod(testPodName)); err != nil {
69+
t.Fatalf(fmt.Sprintf("timed out waiting for wget pod to succeed: %v", err))
70+
}
71+
72+
t.Run("DefaultDeny", HelperDefaultDeny)
73+
74+
resetNetworkPolicy := func() {
75+
n, err := client.CoreV1().Namespaces().Get(namespace, metav1.GetOptions{})
76+
if err != nil {
77+
t.Fatalf("unable to retrieve namespace %v", err)
78+
}
79+
80+
n.ObjectMeta.Annotations = map[string]string{}
81+
n.ObjectMeta.Annotations["net.beta.kubernetes.io/network-policy"] = defaultAllowNetworkPolicy
82+
_, err = client.CoreV1().Namespaces().Update(n)
83+
if err != nil {
84+
t.Fatalf("unable to reset namespace network policy%v", err)
85+
}
86+
}
87+
defer resetNetworkPolicy()
88+
89+
t.Run("NetworkPolicy", HelperPolicy)
90+
}
91+
92+
func HelperDefaultDeny(t *testing.T) {
93+
//
94+
// 3. set DefaultDeny policy
95+
var n *v1.Namespace
96+
n, err := client.CoreV1().Namespaces().Get(namespace, metav1.GetOptions{})
97+
if err != nil {
98+
t.Fatalf("unable to retrieve namespace %v", err)
99+
}
100+
101+
n.ObjectMeta.Annotations = map[string]string{}
102+
n.ObjectMeta.Annotations["net.beta.kubernetes.io/network-policy"] = defaultDenyNetworkPolicy
103+
_, err = client.CoreV1().Namespaces().Update(n)
104+
if err != nil {
105+
t.Fatalf("unable to set namespace network policy defaultdeny%v", err)
106+
}
107+
108+
//
109+
// 4. create a wget pod that fails to hit nginx service
110+
testPodName := fmt.Sprintf("%s-%s", "wget-pod-nt", utilrand.String(5))
111+
wgetPodNT.ObjectMeta.Name = testPodName
112+
_, err = client.CoreV1().Pods(namespace).Create(wgetPodNT)
113+
if err != nil {
114+
t.Fatal(err)
115+
}
116+
117+
if err := retry(10, time.Second*10, getFailedPod(testPodName)); err != nil {
118+
t.Fatalf(fmt.Sprintf("timed out waiting for wget pod to fail: %v", err))
119+
}
120+
}
121+
122+
func HelperPolicy(t *testing.T) {
123+
//
124+
// 5. create NetworkPolicy that allows `allow=access`
125+
npi, _, err := api.Codecs.UniversalDecoder().Decode(netPolicy, nil, &v1beta1.NetworkPolicy{})
126+
if err != nil {
127+
t.Fatalf("unable to decode network policy manifest: %v", err)
128+
}
129+
130+
np, ok := npi.(*v1beta1.NetworkPolicy)
131+
if !ok {
132+
t.Fatalf("expected manifest to decode into *api.networkpolicy, got %T", npi)
133+
}
134+
135+
httpRestClient := client.ExtensionsV1beta1().RESTClient()
136+
uri := fmt.Sprintf("/apis/%s/%s/namespaces/%s/%s",
137+
strings.ToLower("extensions"),
138+
strings.ToLower("v1beta1"),
139+
strings.ToLower(namespace),
140+
strings.ToLower("NetworkPolicies"))
141+
142+
result := httpRestClient.Post().RequestURI(uri).Body(np).Do()
143+
if result.Error() != nil {
144+
t.Fatal(result.Error())
145+
}
146+
defer func() {
147+
uri = fmt.Sprintf("/apis/%s/%s/namespaces/%s/%s/%s",
148+
strings.ToLower("extensions"),
149+
strings.ToLower("v1beta1"),
150+
strings.ToLower(namespace),
151+
strings.ToLower("NetworkPolicies"),
152+
strings.ToLower(np.ObjectMeta.Name))
153+
154+
result = httpRestClient.Delete().RequestURI(uri).Do()
155+
if result.Error() != nil {
156+
t.Fatal(result.Error())
157+
}
158+
159+
}()
160+
161+
//
162+
// 6. create a wget pod with label `allow=access` that hits the nginx service
163+
testPodName := fmt.Sprintf("%s-%s", "wget-pod-nt", utilrand.String(5))
164+
wgetPodNT.ObjectMeta.Name = testPodName
165+
wgetPodNT.ObjectMeta.Labels = map[string]string{}
166+
wgetPodNT.ObjectMeta.Labels["allow"] = "access"
167+
_, err = client.CoreV1().Pods(namespace).Create(wgetPodNT)
168+
if err != nil {
169+
t.Fatal(err)
170+
}
171+
172+
if err := retry(10, time.Second*10, getPod(testPodName)); err != nil {
173+
t.Fatalf(fmt.Sprintf("timed out waiting for wget pod to succeed: %v", err))
174+
}
175+
176+
//
177+
// 7. create a wget pod with label `allow=cant-access` that fails to the nginx service
178+
testPodName = fmt.Sprintf("%s-%s", "wget-pod-nt", utilrand.String(5))
179+
wgetPodNT.ObjectMeta.Name = testPodName
180+
wgetPodNT.ObjectMeta.Labels["allow"] = "cant-access"
181+
_, err = client.CoreV1().Pods(namespace).Create(wgetPodNT)
182+
if err != nil {
183+
t.Fatal(err)
184+
}
185+
186+
if err := retry(10, time.Second*10, getFailedPod(testPodName)); err != nil {
187+
t.Fatalf(fmt.Sprintf("timed out waiting for wget pod to fail: %v", err))
188+
}
189+
}
190+
191+
func getNginxPod() error {
192+
l, err := client.CoreV1().Pods(namespace).List(metav1.ListOptions{LabelSelector: "app=nginx"})
193+
if err != nil || len(l.Items) == 0 {
194+
return fmt.Errorf("couldn't list pods: %v", err)
195+
}
196+
197+
// take the first pod
198+
p := &l.Items[0]
199+
200+
if p.Status.Phase != v1.PodRunning {
201+
return fmt.Errorf("pod not yet running: %v", p.Status.Phase)
202+
}
203+
return nil
204+
}
205+
206+
func getPod(name string) func() error {
207+
return func() error {
208+
p, err := client.CoreV1().Pods(namespace).Get(name, metav1.GetOptions{})
209+
if err != nil {
210+
return fmt.Errorf("couldn't get pod: %v", err)
211+
}
212+
if p.Status.Phase != v1.PodSucceeded {
213+
return fmt.Errorf("pod did not succeed: %v", p.Status.Phase)
214+
}
215+
return nil
216+
}
217+
}
218+
219+
func getFailedPod(name string) func() error {
220+
return func() error {
221+
p, err := client.CoreV1().Pods(namespace).Get(name, metav1.GetOptions{})
222+
if err != nil {
223+
return fmt.Errorf("couldn't get pod: %v", err)
224+
}
225+
if p.Status.Phase != v1.PodFailed {
226+
return fmt.Errorf("pod did not fail: %v", p.Status.Phase)
227+
}
228+
return nil
229+
}
230+
}
231+
232+
var nginxDepNT = []byte(`apiVersion: apps/v1beta1
233+
kind: Deployment
234+
metadata:
235+
name: nginx-deployment-nt
236+
spec:
237+
replicas: 3
238+
template:
239+
metadata:
240+
labels:
241+
app: nginx
242+
spec:
243+
containers:
244+
- name: nginx
245+
image: nginx:1.8
246+
ports:
247+
- containerPort: 80
248+
`)
249+
250+
var wgetPodNT = &v1.Pod{
251+
ObjectMeta: metav1.ObjectMeta{
252+
Namespace: namespace,
253+
},
254+
Spec: v1.PodSpec{
255+
Containers: []v1.Container{
256+
{
257+
Name: "wget-container",
258+
Image: "busybox:1.26",
259+
Command: []string{"wget", "--timeout", "5", "nginx-service-nt"},
260+
},
261+
},
262+
RestartPolicy: v1.RestartPolicyNever,
263+
},
264+
}
265+
266+
var nginxSVCNT = []byte(`apiVersion: v1
267+
kind: Service
268+
metadata:
269+
name: nginx-service-nt
270+
spec:
271+
selector:
272+
app: nginx
273+
ports:
274+
- protocol: TCP
275+
port: 80
276+
targetPort: 80
277+
`)
278+
279+
var defaultDenyNetworkPolicy = `{"ingress":{"isolation":"DefaultDeny"}}`
280+
var defaultAllowNetworkPolicy = `{"ingress":{"isolation":""}}`
281+
282+
var netPolicy = []byte(`kind: NetworkPolicy
283+
apiVersion: extensions/v1beta1
284+
metadata:
285+
name: access-nginx
286+
spec:
287+
podSelector:
288+
matchLabels:
289+
app: nginx
290+
ingress:
291+
- from:
292+
- podSelector:
293+
matchLabels:
294+
allow: access
295+
`)

‎hack/multi-node/bootkube-up‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ fi
2020

2121
# Render assets
2222
if [ ! -d "cluster" ]; then
23-
../../_output/bin/${local_os}/bootkube render --asset-dir=cluster --api-servers=https://172.17.4.101:443 ${etcd_render_flags}
23+
../../_output/bin/${local_os}/bootkube render --asset-dir=cluster --api-servers=https://172.17.4.101:443 ${etcd_render_flags} --experimental-calico-network-policy
2424
cp user-data.sample cluster/user-data-worker
2525
cp user-data.sample cluster/user-data-controller
2626
sed -i.bak -e '/node-role.kubernetes.io\/master/d' cluster/user-data-worker

‎hack/multi-node/user-data.sample‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,10 @@ coreos:
1212
Environment=KUBELET_IMAGE_TAG=v1.6.4_coreos.0
1313
Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \
1414
--volume var-lib-cni,kind=host,source=/var/lib/cni \
15-
--mount volume=var-lib-cni,target=/var/lib/cni"
15+
--volume opt-cni-bin,kind=host,source=/opt/cni/bin \
16+
--mount volume=var-lib-cni,target=/var/lib/cni \
17+
--mount volume=opt-cni-bin,target=/opt/cni/bin"
18+
ExecStartPre=/bin/mkdir -p /opt/cni/bin
1619
ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
1720
ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
1821
ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets

‎hack/quickstart/init-master.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ function init_master_node() {
5757
fi
5858

5959
# Render cluster assets
60-
/home/${REMOTE_USER}/bootkube render --asset-dir=/home/${REMOTE_USER}/assets ${etcd_render_flags} \
60+
/home/${REMOTE_USER}/bootkube render --asset-dir=/home/${REMOTE_USER}/assets ${etcd_render_flags} --experimental-calico-network-policy \
6161
--api-servers=https://${COREOS_PUBLIC_IPV4}:443,https://${COREOS_PRIVATE_IPV4}:443
6262

6363
# Move the local kubeconfig into expected location

0 commit comments

Comments
 (0)