Skip to content

Commit d7e0a51

Browse files
aojeaaojea
committed
add presubmit jobs to avoid regressions on konnectivity
1 parent fae2064 commit d7e0a51

File tree

1 file changed

+220
-0
lines changed

1 file changed

+220
-0
lines changed

‎.github/workflows/e2e.yaml

Lines changed: 220 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,220 @@
1+
name: e2e
2+
3+
on:
4+
push:
5+
branches:
6+
- 'master'
7+
tags:
8+
- 'v*'
9+
pull_request:
10+
branches: [ master ]
11+
workflow_dispatch:
12+
13+
env:
14+
GO_VERSION: "1.21.4"
15+
K8S_VERSION: "v1.27.3"
16+
KIND_CLUSTER_NAME: "kind"
17+
18+
jobs:
19+
build:
20+
name: build
21+
runs-on: ubuntu-20.04
22+
steps:
23+
- name: Set up Go
24+
uses: actions/setup-go@v2
25+
with:
26+
go-version: ${{ env.GO_VERSION }}
27+
id: go
28+
29+
- name: Check out code
30+
uses: actions/checkout@v2
31+
32+
- name: Build
33+
run: |
34+
mkdir _output
35+
36+
docker build -t gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master -f artifacts/images/agent-build.Dockerfile .
37+
docker save gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master > _output/konnectivity-agent.tar
38+
docker build -t gcr.io/k8s-staging-kas-network-proxy/proxy-server:master -f artifacts/images/server-build.Dockerfile .
39+
docker save gcr.io/k8s-staging-kas-network-proxy/proxy-server:master > > _output/konnectivity-server.tar
40+
41+
- uses: actions/upload-artifact@v2
42+
with:
43+
name: konnectivity-server
44+
path: _output/konnectivity-server.tar
45+
46+
- uses: actions/upload-artifact@v2
47+
with:
48+
name: konnectivity-agent
49+
path: _output/konnectivity-agent.tar
50+
e2e:
51+
name: e2e
52+
runs-on: ubuntu-20.04
53+
timeout-minutes: 100
54+
needs:
55+
- build
56+
strategy:
57+
fail-fast: false
58+
matrix:
59+
ipFamily: ["ipv4", "ipv6", "dual"]
60+
env:
61+
JOB_NAME: "kindnetd-e2e-${{ matrix.ipFamily }}"
62+
IP_FAMILY: ${{ matrix.ipFamily }}
63+
steps:
64+
- name: Check out code
65+
uses: actions/checkout@v2
66+
67+
- name: Enable ipv4 and ipv6 forwarding
68+
run: |
69+
sudo sysctl -w net.ipv6.conf.all.forwarding=1
70+
sudo sysctl -w net.ipv4.ip_forward=1
71+
72+
- name: Set up environment (download dependencies)
73+
run: |
74+
TMP_DIR=$(mktemp -d)
75+
# Test binaries
76+
curl -L https://dl.k8s.io/${{ env.K8S_VERSION }}/kubernetes-test-linux-amd64.tar.gz -o ${TMP_DIR}/kubernetes-test-linux-amd64.tar.gz
77+
tar xvzf ${TMP_DIR}/kubernetes-test-linux-amd64.tar.gz \
78+
--directory ${TMP_DIR} \
79+
--strip-components=3 kubernetes/test/bin/ginkgo kubernetes/test/bin/e2e.test
80+
# kubectl
81+
curl -L https://dl.k8s.io/${{ env.K8S_VERSION }}/bin/linux/amd64/kubectl -o ${TMP_DIR}/kubectl
82+
# kind
83+
curl -Lo ${TMP_DIR}/kind https://kind.sigs.k8s.io/dl/v0.17.0/kind-linux-amd64
84+
# Install
85+
sudo cp ${TMP_DIR}/ginkgo /usr/local/bin/ginkgo
86+
sudo cp ${TMP_DIR}/e2e.test /usr/local/bin/e2e.test
87+
sudo cp ${TMP_DIR}/kubectl /usr/local/bin/kubectl
88+
sudo cp ${TMP_DIR}/kind /usr/local/bin/kind
89+
sudo chmod +x /usr/local/bin/*
90+
91+
- name: Create multi node cluster
92+
run: |
93+
# output_dir
94+
mkdir -p _artifacts
95+
# create cluster
96+
cat <<EOF | /usr/local/bin/kind create cluster \
97+
--name ${{ env.KIND_CLUSTER_NAME}} \
98+
--image kindest/node:${{ env.K8S_VERSION }} \
99+
-v7 --wait 1m --retain --config=-
100+
kind: Cluster
101+
apiVersion: kind.x-k8s.io/v1alpha4
102+
networking:
103+
ipFamily: ${IP_FAMILY}
104+
nodes:
105+
- role: control-plane
106+
kubeadmConfigPatches:
107+
- |
108+
kind: ClusterConfiguration
109+
apiServer:
110+
extraArgs:
111+
"egress-selector-config-file": "/etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml"
112+
extraVolumes:
113+
- name: egress-selector-config-file
114+
hostPath: "/etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml"
115+
mountPath: "/etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml"
116+
readOnly: true
117+
- name: konnectivity-server
118+
hostPath: "/etc/kubernetes/konnectivity-server"
119+
mountPath: "/etc/kubernetes/konnectivity-server"
120+
readOnly: true
121+
extraMounts:
122+
- hostPath: ./examples/kind/egress_selector_configuration.yaml
123+
containerPath: /etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml
124+
EOF
125+
# dump the kubeconfig for later
126+
/usr/local/bin/kind get kubeconfig --name ${{ env.KIND_CLUSTER_NAME}} > _artifacts/kubeconfig.conf
127+
128+
- uses: actions/download-artifact@v2
129+
with:
130+
name: konnectivity-server
131+
132+
- uses: actions/download-artifact@v2
133+
with:
134+
name: konnectivity-agent
135+
136+
- name: Install konnectivity
137+
run: |
138+
# preload konnectivity images
139+
docker load --input konnectivity-server.tar
140+
docker load --input konnectivity-agent.tar
141+
/usr/local/bin/kind load docker-image gcr.io/k8s-staging-kas-network-proxy/proxy-server:master --name ${{ env.KIND_CLUSTER_NAME}}
142+
/usr/local/bin/kind load docker-image gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master --name ${{ env.KIND_CLUSTER_NAME}}
143+
kubectl apply -f examples/kind/konnectivity-server.yaml
144+
kubectl apply -f examples/kind/konnectivity-agent-ds.yaml
145+
146+
- name: Get Cluster status
147+
run: |
148+
# wait network is ready
149+
sleep 5
150+
/usr/local/bin/kubectl get nodes -o wide
151+
/usr/local/bin/kubectl get pods -A
152+
/usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods --namespace=kube-system -l k8s-app=kube-dns
153+
# smoke test
154+
/usr/local/bin/kubectl run test --image httpd:2
155+
/usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods test
156+
/usr/local/bin/kubectl logs test
157+
158+
- name: Workaround CoreDNS for IPv6 airgapped
159+
if: ${{ matrix.ipFamily == 'ipv6' }}
160+
run: |
161+
# Patch CoreDNS to work in Github CI
162+
# 1. Github CI doesn´t offer IPv6 connectivity, so CoreDNS should be configured
163+
# to work in an offline environment:
164+
# https://github.com/coredns/coredns/issues/2494#issuecomment-457215452
165+
# 2. Github CI adds following domains to resolv.conf search field:
166+
# .net.
167+
# CoreDNS should handle those domains and answer with NXDOMAIN instead of SERVFAIL
168+
# otherwise pods stops trying to resolve the domain.
169+
# Get the current config
170+
original_coredns=$(/usr/local/bin/kubectl get -oyaml -n=kube-system configmap/coredns)
171+
echo "Original CoreDNS config:"
172+
echo "${original_coredns}"
173+
# Patch it
174+
fixed_coredns=$(
175+
printf '%s' "${original_coredns}" | sed \
176+
-e 's/^.*kubernetes cluster\.local/& net/' \
177+
-e '/^.*upstream$/d' \
178+
-e '/^.*fallthrough.*$/d' \
179+
-e '/^.*forward . \/etc\/resolv.conf$/d' \
180+
-e '/^.*loop$/d' \
181+
)
182+
echo "Patched CoreDNS config:"
183+
echo "${fixed_coredns}"
184+
printf '%s' "${fixed_coredns}" | /usr/local/bin/kubectl apply -f -
185+
186+
- name: Run tests
187+
run: |
188+
export KUBERNETES_CONFORMANCE_TEST='y'
189+
export E2E_REPORT_DIR=${PWD}/_artifacts
190+
191+
# Run tests
192+
/usr/local/bin/ginkgo --nodes=25 \
193+
--focus="\[Conformance\]" \
194+
--skip="Feature|Federation|machinery|PerformanceDNS|DualStack|Disruptive|Serial|Slow|KubeProxy|LoadBalancer|GCE|Netpol|NetworkPolicy|NodeConformance" \
195+
/usr/local/bin/e2e.test \
196+
-- \
197+
--kubeconfig=${PWD}/_artifacts/kubeconfig.conf \
198+
--provider=local \
199+
--dump-logs-on-failure=false \
200+
--report-dir=${E2E_REPORT_DIR} \
201+
--disable-log-dump=true
202+
203+
- name: Upload Junit Reports
204+
if: always()
205+
uses: actions/upload-artifact@v2
206+
with:
207+
name: kind-junit-${{ env.JOB_NAME }}-${{ github.run_id }}
208+
path: './_artifacts/*.xml'
209+
210+
- name: Export logs
211+
if: always()
212+
run: |
213+
/usr/local/bin/kind export logs --name ${KIND_CLUSTER_NAME} --loglevel=debug ./_artifacts/logs
214+
215+
- name: Upload logs
216+
if: always()
217+
uses: actions/upload-artifact@v2
218+
with:
219+
name: kind-logs-${{ env.JOB_NAME }}-${{ github.run_id }}
220+
path: ./_artifacts/logs

0 commit comments

Comments
 (0)