Skip to content

Commit 460b51d

Browse files
jsturtevantjsturtevant
committed
Add Windows e2e
1 parent e753a32 commit 460b51d

File tree

15 files changed

+642
-16
lines changed

15 files changed

+642
-16
lines changed

‎go.mod‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,13 @@ require (
2222
github.com/onsi/gomega v1.10.3
2323
github.com/pkg/errors v0.9.1
2424
github.com/prometheus/client_golang v1.8.0
25-
github.com/prometheus/common v0.14.0
2625
github.com/spf13/pflag v1.0.5
2726
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.13.0
2827
go.opentelemetry.io/otel v0.13.0
2928
go.opentelemetry.io/otel/exporters/metric/prometheus v0.13.0
3029
go.opentelemetry.io/otel/exporters/trace/jaeger v0.13.0
3130
go.opentelemetry.io/otel/sdk v0.13.0
3231
golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9
33-
golang.org/x/tools v0.0.0-20200904185747-39188db58858
3432
k8s.io/api v0.17.14
3533
k8s.io/apimachinery v0.17.14
3634
k8s.io/client-go v0.17.14

‎templates/cluster-template-windows.yaml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -259,12 +259,15 @@ spec:
259259
path: c:/k/azure.json
260260
permissions: "0644"
261261
- content: |
262+
# required as a work around for Flannel and Wins bugs
263+
# https://github.com/coreos/flannel/issues/1359
264+
# https://github.com/kubernetes-sigs/sig-windows-tools/issues/103#issuecomment-709426828
262265
ipmo C:\k\debug\hns.psm1;
263266
New-HnsNetwork -Type Overlay -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name "External" -AdapterName "Ethernet 2" -SubnetPolicies @(@{Type = "VSID"; VSID = 9999; })
264267
path: C:\create-external-network.ps1
265268
permissions: "0744"
266269
- content: |
267-
[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String("${AZURE_SSH_PUBLIC_KEY_B64:=''}")) > C:\ProgramData\ssh\administrators_authorized_keys
270+
[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String("${AZURE_SSH_PUBLIC_KEY_B64:=''}")) | Add-Content C:\ProgramData\ssh\administrators_authorized_keys
268271
icacls C:\ProgramData\ssh\administrators_authorized_keys /remove "NT AUTHORITY\Authenticated Users"
269272
icacls C:\ProgramData\ssh\administrators_authorized_keys /inheritance:r
270273
icacls C:\ProgramData\ssh\administrators_authorized_keys /grant SYSTEM:`(F`)

‎templates/flavors/windows/machine-deployment-windows.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ spec:
7676
- path: C:\configure-ssh.ps1
7777
permissions: "0744"
7878
content: |
79-
[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String("${AZURE_SSH_PUBLIC_KEY_B64:=''}")) > C:\ProgramData\ssh\administrators_authorized_keys
79+
[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String("${AZURE_SSH_PUBLIC_KEY_B64:=''}")) | Add-Content C:\ProgramData\ssh\administrators_authorized_keys
8080
icacls C:\ProgramData\ssh\administrators_authorized_keys /remove "NT AUTHORITY\Authenticated Users"
8181
icacls C:\ProgramData\ssh\administrators_authorized_keys /inheritance:r
8282
icacls C:\ProgramData\ssh\administrators_authorized_keys /grant SYSTEM:`(F`)

‎templates/test/cluster-template-prow-windows.yaml‎

Lines changed: 332 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,332 @@
1+
apiVersion: cluster.x-k8s.io/v1alpha3
2+
kind: Cluster
3+
metadata:
4+
labels:
5+
cni: ${CLUSTER_NAME}-crs-0
6+
name: ${CLUSTER_NAME}
7+
namespace: default
8+
spec:
9+
clusterNetwork:
10+
pods:
11+
cidrBlocks:
12+
- 10.244.0.0/16
13+
controlPlaneRef:
14+
apiVersion: controlplane.cluster.x-k8s.io/v1alpha3
15+
kind: KubeadmControlPlane
16+
name: ${CLUSTER_NAME}-control-plane
17+
infrastructureRef:
18+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
19+
kind: AzureCluster
20+
name: ${CLUSTER_NAME}
21+
---
22+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
23+
kind: AzureCluster
24+
metadata:
25+
name: ${CLUSTER_NAME}
26+
namespace: default
27+
spec:
28+
additionalTags:
29+
creationTimestamp: ${TIMESTAMP}
30+
jobName: ${JOB_NAME}
31+
location: ${AZURE_LOCATION}
32+
networkSpec:
33+
vnet:
34+
name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet}
35+
resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}}
36+
subscriptionID: ${AZURE_SUBSCRIPTION_ID}
37+
---
38+
apiVersion: controlplane.cluster.x-k8s.io/v1alpha3
39+
kind: KubeadmControlPlane
40+
metadata:
41+
name: ${CLUSTER_NAME}-control-plane
42+
namespace: default
43+
spec:
44+
infrastructureTemplate:
45+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
46+
kind: AzureMachineTemplate
47+
name: ${CLUSTER_NAME}-control-plane
48+
kubeadmConfigSpec:
49+
clusterConfiguration:
50+
apiServer:
51+
extraArgs:
52+
cloud-config: /etc/kubernetes/azure.json
53+
cloud-provider: azure
54+
extraVolumes:
55+
- hostPath: /etc/kubernetes/azure.json
56+
mountPath: /etc/kubernetes/azure.json
57+
name: cloud-config
58+
readOnly: true
59+
timeoutForControlPlane: 20m
60+
controllerManager:
61+
extraArgs:
62+
allocate-node-cidrs: "true"
63+
cloud-config: /etc/kubernetes/azure.json
64+
cloud-provider: azure
65+
cluster-name: ${CLUSTER_NAME}
66+
configure-cloud-routes: "false"
67+
extraVolumes:
68+
- hostPath: /etc/kubernetes/azure.json
69+
mountPath: /etc/kubernetes/azure.json
70+
name: cloud-config
71+
readOnly: true
72+
etcd:
73+
local:
74+
dataDir: /var/lib/etcddisk/etcd
75+
diskSetup:
76+
filesystems:
77+
- device: /dev/disk/azure/scsi1/lun0
78+
extraOpts:
79+
- -E
80+
- lazy_itable_init=1,lazy_journal_init=1
81+
filesystem: ext4
82+
label: etcd_disk
83+
- device: ephemeral0.1
84+
filesystem: ext4
85+
label: ephemeral0
86+
replaceFS: ntfs
87+
partitions:
88+
- device: /dev/disk/azure/scsi1/lun0
89+
layout: true
90+
overwrite: false
91+
tableType: gpt
92+
files:
93+
- contentFrom:
94+
secret:
95+
key: control-plane-azure.json
96+
name: ${CLUSTER_NAME}-control-plane-azure-json
97+
owner: root:root
98+
path: /etc/kubernetes/azure.json
99+
permissions: "0644"
100+
initConfiguration:
101+
nodeRegistration:
102+
kubeletExtraArgs:
103+
cloud-config: /etc/kubernetes/azure.json
104+
cloud-provider: azure
105+
name: '{{ ds.meta_data["local_hostname"] }}'
106+
joinConfiguration:
107+
nodeRegistration:
108+
kubeletExtraArgs:
109+
cloud-config: /etc/kubernetes/azure.json
110+
cloud-provider: azure
111+
name: '{{ ds.meta_data["local_hostname"] }}'
112+
mounts:
113+
- - LABEL=etcd_disk
114+
- /var/lib/etcddisk
115+
useExperimentalRetryJoin: true
116+
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
117+
version: ${KUBERNETES_VERSION}
118+
---
119+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
120+
kind: AzureMachineTemplate
121+
metadata:
122+
name: ${CLUSTER_NAME}-control-plane
123+
namespace: default
124+
spec:
125+
template:
126+
spec:
127+
dataDisks:
128+
- diskSizeGB: 256
129+
lun: 0
130+
nameSuffix: etcddisk
131+
location: ${AZURE_LOCATION}
132+
osDisk:
133+
diskSizeGB: 128
134+
managedDisk:
135+
storageAccountType: Premium_LRS
136+
osType: Linux
137+
sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
138+
vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE}
139+
---
140+
apiVersion: cluster.x-k8s.io/v1alpha3
141+
kind: MachineDeployment
142+
metadata:
143+
name: ${CLUSTER_NAME}-md-0
144+
namespace: default
145+
spec:
146+
clusterName: ${CLUSTER_NAME}
147+
replicas: ${WORKER_MACHINE_COUNT}
148+
selector:
149+
matchLabels: null
150+
template:
151+
spec:
152+
bootstrap:
153+
configRef:
154+
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
155+
kind: KubeadmConfigTemplate
156+
name: ${CLUSTER_NAME}-md-0
157+
clusterName: ${CLUSTER_NAME}
158+
infrastructureRef:
159+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
160+
kind: AzureMachineTemplate
161+
name: ${CLUSTER_NAME}-md-0
162+
version: ${KUBERNETES_VERSION}
163+
---
164+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
165+
kind: AzureMachineTemplate
166+
metadata:
167+
name: ${CLUSTER_NAME}-md-0
168+
namespace: default
169+
spec:
170+
template:
171+
spec:
172+
location: ${AZURE_LOCATION}
173+
osDisk:
174+
diskSizeGB: 128
175+
managedDisk:
176+
storageAccountType: Premium_LRS
177+
osType: Linux
178+
sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
179+
vmSize: ${AZURE_NODE_MACHINE_TYPE}
180+
---
181+
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
182+
kind: KubeadmConfigTemplate
183+
metadata:
184+
name: ${CLUSTER_NAME}-md-0
185+
namespace: default
186+
spec:
187+
template:
188+
spec:
189+
files:
190+
- contentFrom:
191+
secret:
192+
key: worker-node-azure.json
193+
name: ${CLUSTER_NAME}-md-0-azure-json
194+
owner: root:root
195+
path: /etc/kubernetes/azure.json
196+
permissions: "0644"
197+
joinConfiguration:
198+
nodeRegistration:
199+
kubeletExtraArgs:
200+
cloud-config: /etc/kubernetes/azure.json
201+
cloud-provider: azure
202+
name: '{{ ds.meta_data["local_hostname"] }}'
203+
useExperimentalRetryJoin: true
204+
---
205+
apiVersion: cluster.x-k8s.io/v1alpha3
206+
kind: MachineDeployment
207+
metadata:
208+
name: ${CLUSTER_NAME}-md-win
209+
namespace: default
210+
spec:
211+
clusterName: ${CLUSTER_NAME}
212+
replicas: ${WORKER_MACHINE_COUNT}
213+
selector:
214+
matchLabels: null
215+
template:
216+
spec:
217+
bootstrap:
218+
configRef:
219+
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
220+
kind: KubeadmConfigTemplate
221+
name: ${CLUSTER_NAME}-md-win
222+
clusterName: ${CLUSTER_NAME}
223+
infrastructureRef:
224+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
225+
kind: AzureMachineTemplate
226+
name: ${CLUSTER_NAME}-md-win
227+
version: ${KUBERNETES_VERSION}
228+
---
229+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
230+
kind: AzureMachineTemplate
231+
metadata:
232+
name: ${CLUSTER_NAME}-md-win
233+
namespace: default
234+
spec:
235+
template:
236+
spec:
237+
image:
238+
id: /subscriptions/b9d9436a-0c07-4fe8-b779-2c1030bd7997/resourceGroups/test-cluster-api-images/providers/Microsoft.Compute/images/windowstestvmimage-capz
239+
location: ${AZURE_LOCATION}
240+
osDisk:
241+
diskSizeGB: 128
242+
managedDisk:
243+
storageAccountType: Premium_LRS
244+
osType: Windows
245+
sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
246+
vmSize: ${AZURE_NODE_MACHINE_TYPE}
247+
---
248+
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
249+
kind: KubeadmConfigTemplate
250+
metadata:
251+
name: ${CLUSTER_NAME}-md-win
252+
namespace: default
253+
spec:
254+
template:
255+
spec:
256+
files:
257+
- contentFrom:
258+
secret:
259+
key: worker-node-azure.json
260+
name: ${CLUSTER_NAME}-md-win-azure-json
261+
owner: root:root
262+
path: c:/k/azure.json
263+
permissions: "0644"
264+
- content: |
265+
# required as a work around for Flannel and Wins bugs
266+
# https://github.com/coreos/flannel/issues/1359
267+
# https://github.com/kubernetes-sigs/sig-windows-tools/issues/103#issuecomment-709426828
268+
ipmo C:\k\debug\hns.psm1;
269+
New-HnsNetwork -Type Overlay -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name "External" -AdapterName "Ethernet 2" -SubnetPolicies @(@{Type = "VSID"; VSID = 9999; })
270+
path: C:\create-external-network.ps1
271+
permissions: "0744"
272+
- content: |
273+
[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String("${AZURE_SSH_PUBLIC_KEY_B64:=''}")) | Add-Content C:\ProgramData\ssh\administrators_authorized_keys
274+
icacls C:\ProgramData\ssh\administrators_authorized_keys /remove "NT AUTHORITY\Authenticated Users"
275+
icacls C:\ProgramData\ssh\administrators_authorized_keys /inheritance:r
276+
icacls C:\ProgramData\ssh\administrators_authorized_keys /grant SYSTEM:`(F`)
277+
icacls C:\ProgramData\ssh\administrators_authorized_keys /grant BUILTIN\Administrators:`(F`)
278+
restart-service sshd
279+
path: C:\configure-ssh.ps1
280+
permissions: "0744"
281+
joinConfiguration:
282+
nodeRegistration:
283+
kubeletExtraArgs:
284+
cloud-config: c:/k/azure.json
285+
cloud-provider: azure
286+
pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:1.4.0
287+
name: '{{ ds.meta_data["local_hostname"] }}'
288+
preKubeadmCommands:
289+
- powershell c:\create-external-network.ps1
290+
- powershell c:\configure-ssh.ps1
291+
---
292+
apiVersion: v1
293+
data: ${CNI_RESOURCES_WINDOWS}
294+
kind: ConfigMap
295+
metadata:
296+
name: cni-${CLUSTER_NAME}-crs-0
297+
namespace: default
298+
---
299+
apiVersion: addons.cluster.x-k8s.io/v1alpha3
300+
kind: ClusterResourceSet
301+
metadata:
302+
name: ${CLUSTER_NAME}-crs-0
303+
namespace: default
304+
spec:
305+
clusterSelector:
306+
matchLabels:
307+
cni: ${CLUSTER_NAME}-crs-0
308+
resources:
309+
- kind: ConfigMap
310+
name: cni-${CLUSTER_NAME}-crs-0
311+
strategy: ApplyOnce
312+
---
313+
apiVersion: v1
314+
data: ${WINDOWS_RESOURCES_KUBEPROXY}
315+
kind: ConfigMap
316+
metadata:
317+
name: kubeproxy-${CLUSTER_NAME}-crs-0
318+
namespace: default
319+
---
320+
apiVersion: addons.cluster.x-k8s.io/v1alpha3
321+
kind: ClusterResourceSet
322+
metadata:
323+
name: win-kubeproxy-${CLUSTER_NAME}-crs-0
324+
namespace: default
325+
spec:
326+
clusterSelector:
327+
matchLabels:
328+
win-kubeproxy: ${CLUSTER_NAME}-kubeproxy-crs-0
329+
resources:
330+
- kind: ConfigMap
331+
name: kubeproxy-${CLUSTER_NAME}-crs-0
332+
strategy: ApplyOnce

‎templates/test/prow-windows/cni-resource-set.yaml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
---
2+
apiVersion: v1
3+
data: ${CNI_RESOURCES_WINDOWS}
4+
kind: ConfigMap
5+
metadata:
6+
name: cni-${CLUSTER_NAME}-crs-0
7+
namespace: default
8+
---
9+
apiVersion: addons.cluster.x-k8s.io/v1alpha3
10+
kind: ClusterResourceSet
11+
metadata:
12+
name: ${CLUSTER_NAME}-crs-0
13+
namespace: default
14+
spec:
15+
clusterSelector:
16+
matchLabels:
17+
cni: ${CLUSTER_NAME}-crs-0
18+
resources:
19+
- kind: ConfigMap
20+
name: cni-${CLUSTER_NAME}-crs-0
21+
strategy: ApplyOnce

0 commit comments

Comments
 (0)