Skip to content

Commit 2758760

Browse files
jsturtevantjsturtevant
committed
Fix for crs which drops \\ on windows named pipes
1 parent b895dfe commit 2758760

20 files changed

+1008
-169
lines changed

‎Tiltfile‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -196,13 +196,21 @@ def capz():
196196

197197
k8s_yaml(blob(yaml))
198198

199-
def calico_crs():
199+
def create_crs():
200+
# create config maps
200201
local("kubectl delete configmaps calico-addon --ignore-not-found=true")
201202
local("kubectl create configmap calico-addon --from-file=templates/addons/calico.yaml")
202203
local("kubectl delete configmaps calico-ipv6-addon --ignore-not-found=true")
203204
local("kubectl create configmap calico-ipv6-addon --from-file=templates/addons/calico-ipv6.yaml")
204205
local("kubectl delete configmaps flannel-windows-addon --ignore-not-found=true")
205-
local("kubectl create configmap flannel-windows-addon --from-file=templates/addons/windows/")
206+
207+
# need to set version for kube-proxy on windows.
208+
# This file is processed then reapply \\ due to the named pipes which need to be escaped for a bug in envsubst library
209+
# https://github.com/kubernetes-sigs/cluster-api/issues/4016
210+
os.putenv("KUBERNETES_VERSION", settings.get("kubernetes_version", {}))
211+
local("kubectl create configmap flannel-windows-addon --from-file=templates/addons/windows/ --dry-run=client -o yaml | " + envsubst_cmd + " | sed -e 's/\\\\/\\\\\\\\/' | kubectl apply -f -")
212+
213+
# set up crs
206214
local("kubectl wait --for=condition=Available --timeout=300s -n capi-webhook-system deployment/capi-controller-manager")
207215
local("kubectl apply -f templates/addons/calico-resource-set.yaml")
208216
local("kubectl apply -f templates/addons/flannel-resource-set.yaml")
@@ -267,8 +275,9 @@ def deploy_worker_templates(template, substitutions):
267275
yaml = yaml.replace("${" + substitution + "}", value)
268276

269277
# programmatically define any remaining vars
278+
# "windows" can not be for cluster name because it sets the dns to trademarked name during reconciliation
270279
substitutions = {
271-
"CLUSTER_NAME": flavor + "-template",
280+
"CLUSTER_NAME": flavor.replace("windows", "win") + "-template",
272281
"AZURE_LOCATION": "eastus",
273282
"AZURE_VNET_NAME": flavor + "-template-vnet",
274283
"AZURE_RESOURCE_GROUP": flavor + "-template-rg",
@@ -337,6 +346,6 @@ deploy_capi()
337346

338347
capz()
339348

340-
calico_crs()
349+
create_crs()
341350

342351
flavors()

‎cloud/scope/machine.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121
"encoding/base64"
2222
"encoding/json"
23+
"strings"
2324

2425
"github.com/Azure/go-autorest/autorest/to"
2526
"github.com/go-logr/logr"
@@ -263,11 +264,11 @@ func (m *MachineScope) Name() string {
263264
// Windows Machine names cannot be longer than 15 chars
264265
if m.AzureMachine.Spec.OSDisk.OSType == azure.WindowsOS && len(m.AzureMachine.Name) > 15 {
265266
clustername := m.ClusterName()
266-
if len(m.ClusterName()) > 5 {
267-
clustername = clustername[0:5]
267+
if len(m.ClusterName()) > 9 {
268+
clustername = strings.TrimSuffix(clustername[0:9], "-")
268269
}
269270

270-
return "win-" + clustername + "-" + m.AzureMachine.Name[len(m.AzureMachine.Name)-5:]
271+
return clustername + "-" + m.AzureMachine.Name[len(m.AzureMachine.Name)-5:]
271272
}
272273
return m.AzureMachine.Name
273274
}

‎templates/addons/windows/flannel.yaml‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -706,9 +706,9 @@ spec:
706706
effect: NoSchedule
707707
containers:
708708
- name: kube-flannel
709-
image: sigwindowstools/flannel:0.12.0
709+
image: sigwindowstools/flannel:v0.13.0-nanoserver
710710
command:
711-
- powershell
711+
- pwsh
712712
args:
713713
- -file
714714
- /etc/kube-flannel-windows/run.ps1
@@ -739,6 +739,10 @@ spec:
739739
apiVersion: v1
740740
fieldPath: metadata.namespace
741741
volumes:
742+
- name: wins
743+
hostPath:
744+
path: \\.\pipe\rancher_wins
745+
type: null
742746
- name: opt
743747
hostPath:
744748
path: /opt
@@ -760,7 +764,4 @@ spec:
760764
- name: kubeadm-config
761765
configMap:
762766
name: kubeadm-config
763-
- name: wins
764-
hostPath:
765-
path: \\.\pipe\rancher_wins
766-
type: null
767+

‎templates/addons/windows/kube-proxy-windows.yaml‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,7 @@ data:
1616
yq w -i /host/var/lib/kube-proxy/config.conf featureGates.WinOverlay true
1717
yq w -i /host/var/lib/kube-proxy/config.conf mode "kernelspace"
1818
wins cli process run --path /k/kube-proxy/kube-proxy.exe --args "--v=6 --config=/var/lib/kube-proxy/config.conf --hostname-override=$env:NODE_NAME --feature-gates=WinOverlay=true"
19-
2019
kind: ConfigMap
21-
apiVersion: v1
2220
metadata:
2321
labels:
2422
app: kube-proxy
@@ -44,7 +42,7 @@ spec:
4442
serviceAccountName: kube-proxy
4543
containers:
4644
- command:
47-
- powershell
45+
- pwsh
4846
args:
4947
- -file
5048
- /var/lib/kube-proxy-windows/run-script.ps1
@@ -58,13 +56,13 @@ spec:
5856
valueFrom:
5957
fieldRef:
6058
fieldPath: status.podIP
61-
image: sigwindowstools/kube-proxy:v1.19.1
59+
image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION}-nanoserver
6260
name: kube-proxy
6361
volumeMounts:
64-
- name: host
65-
mountPath: /host
6662
- name: wins
6763
mountPath: \\.\pipe\rancher_wins
64+
- name: host
65+
mountPath: /host
6866
- mountPath: /var/lib/kube-proxy
6967
name: kube-proxy
7068
- mountPath: /var/lib/kube-proxy-windows
@@ -76,6 +74,10 @@ spec:
7674
operator: Exists
7775
- operator: Exists
7876
volumes:
77+
- name: wins
78+
hostPath:
79+
path: \\.\pipe\rancher_wins
80+
type: null
7981
- configMap:
8082
defaultMode: 420
8183
name: kube-proxy-windows
@@ -86,9 +88,5 @@ spec:
8688
- hostPath:
8789
path: /
8890
name: host
89-
- name: wins
90-
hostPath:
91-
path: \\.\pipe\rancher_wins
92-
type: null
9391
updateStrategy:
9492
type: RollingUpdate

‎templates/addons/windows/kustomization.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
apiVersion: kustomize.config.k8s.io/v1beta1
2+
kind: Kustomization
3+
namespace: kube-system
4+
resources:
5+
- flannel.yaml
6+
- kube-proxy-windows.yaml

‎templates/cluster-template-machinepool-windows.yaml‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -254,16 +254,16 @@ spec:
254254
# https://github.com/kubernetes-sigs/sig-windows-tools/issues/103#issuecomment-709426828
255255
ipmo C:\k\debug\hns.psm1;
256256
New-HnsNetwork -Type Overlay -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name "External" -AdapterName "Ethernet 2" -SubnetPolicies @(@{Type = "VSID"; VSID = 9999; })
257-
path: C:\create-external-network.ps1
257+
path: C:/create-external-network.ps1
258258
permissions: "0744"
259259
- content: |
260-
[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String("${AZURE_SSH_PUBLIC_KEY_B64:=''}")) > C:\ProgramData\ssh\administrators_authorized_keys
261-
icacls C:\ProgramData\ssh\administrators_authorized_keys /remove "NT AUTHORITY\Authenticated Users"
262-
icacls C:\ProgramData\ssh\administrators_authorized_keys /inheritance:r
263-
icacls C:\ProgramData\ssh\administrators_authorized_keys /grant SYSTEM:`(F`)
264-
icacls C:\ProgramData\ssh\administrators_authorized_keys /grant BUILTIN\Administrators:`(F`)
260+
[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String("${AZURE_SSH_PUBLIC_KEY_B64:=''}")) | Add-Content C:/ProgramData/ssh/administrators_authorized_keys
261+
icacls C:/ProgramData/ssh/administrators_authorized_keys /remove "NT AUTHORITY\Authenticated Users"
262+
icacls C:/ProgramData/ssh/administrators_authorized_keys /inheritance:r
263+
icacls C:/ProgramData/ssh/administrators_authorized_keys /grant SYSTEM:F
264+
icacls C:/ProgramData/ssh/administrators_authorized_keys /grant BUILTIN\Administrators:F
265265
restart-service sshd
266-
path: C:\configure-ssh.ps1
266+
path: C:/configure-ssh.ps1
267267
permissions: "0744"
268268
joinConfiguration:
269269
nodeRegistration:
@@ -273,5 +273,5 @@ spec:
273273
pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:1.4.1
274274
name: '{{ ds.meta_data["local_hostname"] }}'
275275
preKubeadmCommands:
276-
- powershell c:\create-external-network.ps1
277-
- powershell c:\configure-ssh.ps1
276+
- powershell c:/create-external-network.ps1
277+
- powershell c:/configure-ssh.ps1

‎templates/cluster-template-windows.yaml‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -264,16 +264,16 @@ spec:
264264
# https://github.com/kubernetes-sigs/sig-windows-tools/issues/103#issuecomment-709426828
265265
ipmo C:\k\debug\hns.psm1;
266266
New-HnsNetwork -Type Overlay -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name "External" -AdapterName "Ethernet 2" -SubnetPolicies @(@{Type = "VSID"; VSID = 9999; })
267-
path: C:\create-external-network.ps1
267+
path: C:/create-external-network.ps1
268268
permissions: "0744"
269269
- content: |
270-
[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String("${AZURE_SSH_PUBLIC_KEY_B64:=''}")) | Add-Content C:\ProgramData\ssh\administrators_authorized_keys
271-
icacls C:\ProgramData\ssh\administrators_authorized_keys /remove "NT AUTHORITY\Authenticated Users"
272-
icacls C:\ProgramData\ssh\administrators_authorized_keys /inheritance:r
273-
icacls C:\ProgramData\ssh\administrators_authorized_keys /grant SYSTEM:`(F`)
274-
icacls C:\ProgramData\ssh\administrators_authorized_keys /grant BUILTIN\Administrators:`(F`)
270+
[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String("${AZURE_SSH_PUBLIC_KEY_B64:=''}")) | Add-Content C:/ProgramData/ssh/administrators_authorized_keys
271+
icacls C:/ProgramData/ssh/administrators_authorized_keys /remove "NT AUTHORITY\Authenticated Users"
272+
icacls C:/ProgramData/ssh/administrators_authorized_keys /inheritance:r
273+
icacls C:/ProgramData/ssh/administrators_authorized_keys /grant SYSTEM:F
274+
icacls C:/ProgramData/ssh/administrators_authorized_keys /grant BUILTIN\Administrators:F
275275
restart-service sshd
276-
path: C:\configure-ssh.ps1
276+
path: C:/configure-ssh.ps1
277277
permissions: "0744"
278278
joinConfiguration:
279279
nodeRegistration:
@@ -283,5 +283,5 @@ spec:
283283
pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:1.4.1
284284
name: '{{ ds.meta_data["local_hostname"] }}'
285285
preKubeadmCommands:
286-
- powershell c:\create-external-network.ps1
287-
- powershell c:\configure-ssh.ps1
286+
- powershell c:/create-external-network.ps1
287+
- powershell c:/configure-ssh.ps1

‎templates/flavors/machinepool-windows/machine-pool-deployment-windows.yaml‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,8 @@ metadata:
4343
name: "${CLUSTER_NAME}-mp-win"
4444
spec:
4545
preKubeadmCommands:
46-
- powershell c:\create-external-network.ps1
47-
- powershell c:\configure-ssh.ps1
46+
- powershell c:/create-external-network.ps1
47+
- powershell c:/configure-ssh.ps1
4848
joinConfiguration:
4949
nodeRegistration:
5050
name: '{{ ds.meta_data["local_hostname"] }}'
@@ -60,20 +60,20 @@ spec:
6060
owner: root:root
6161
path: c:/k/azure.json
6262
permissions: "0644"
63-
- path: C:\create-external-network.ps1
63+
- path: C:/create-external-network.ps1
6464
permissions: "0744"
6565
content: |
6666
# required as a work around for Flannel and Wins bugs
6767
# https://github.com/coreos/flannel/issues/1359
6868
# https://github.com/kubernetes-sigs/sig-windows-tools/issues/103#issuecomment-709426828
6969
ipmo C:\k\debug\hns.psm1;
7070
New-HnsNetwork -Type Overlay -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name "External" -AdapterName "Ethernet 2" -SubnetPolicies @(@{Type = "VSID"; VSID = 9999; })
71-
- path: C:\configure-ssh.ps1
71+
- path: C:/configure-ssh.ps1
7272
permissions: "0744"
7373
content: |
74-
[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String("${AZURE_SSH_PUBLIC_KEY_B64:=''}")) > C:\ProgramData\ssh\administrators_authorized_keys
75-
icacls C:\ProgramData\ssh\administrators_authorized_keys /remove "NT AUTHORITY\Authenticated Users"
76-
icacls C:\ProgramData\ssh\administrators_authorized_keys /inheritance:r
77-
icacls C:\ProgramData\ssh\administrators_authorized_keys /grant SYSTEM:`(F`)
78-
icacls C:\ProgramData\ssh\administrators_authorized_keys /grant BUILTIN\Administrators:`(F`)
74+
[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String("${AZURE_SSH_PUBLIC_KEY_B64:=''}")) | Add-Content C:/ProgramData/ssh/administrators_authorized_keys
75+
icacls C:/ProgramData/ssh/administrators_authorized_keys /remove "NT AUTHORITY\Authenticated Users"
76+
icacls C:/ProgramData/ssh/administrators_authorized_keys /inheritance:r
77+
icacls C:/ProgramData/ssh/administrators_authorized_keys /grant SYSTEM:F
78+
icacls C:/ProgramData/ssh/administrators_authorized_keys /grant BUILTIN\Administrators:F
7979
restart-service sshd

‎templates/flavors/windows/machine-deployment-windows.yaml‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,8 @@ spec:
4848
template:
4949
spec:
5050
preKubeadmCommands:
51-
- powershell c:\create-external-network.ps1
52-
- powershell c:\configure-ssh.ps1
51+
- powershell c:/create-external-network.ps1
52+
- powershell c:/configure-ssh.ps1
5353
joinConfiguration:
5454
nodeRegistration:
5555
name: '{{ ds.meta_data["local_hostname"] }}'
@@ -65,20 +65,20 @@ spec:
6565
owner: root:root
6666
path: c:/k/azure.json
6767
permissions: "0644"
68-
- path: C:\create-external-network.ps1
68+
- path: C:/create-external-network.ps1
6969
permissions: "0744"
7070
content: |
7171
# required as a work around for Flannel and Wins bugs
7272
# https://github.com/coreos/flannel/issues/1359
7373
# https://github.com/kubernetes-sigs/sig-windows-tools/issues/103#issuecomment-709426828
7474
ipmo C:\k\debug\hns.psm1;
7575
New-HnsNetwork -Type Overlay -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name "External" -AdapterName "Ethernet 2" -SubnetPolicies @(@{Type = "VSID"; VSID = 9999; })
76-
- path: C:\configure-ssh.ps1
76+
- path: C:/configure-ssh.ps1
7777
permissions: "0744"
7878
content: |
79-
[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String("${AZURE_SSH_PUBLIC_KEY_B64:=''}")) | Add-Content C:\ProgramData\ssh\administrators_authorized_keys
80-
icacls C:\ProgramData\ssh\administrators_authorized_keys /remove "NT AUTHORITY\Authenticated Users"
81-
icacls C:\ProgramData\ssh\administrators_authorized_keys /inheritance:r
82-
icacls C:\ProgramData\ssh\administrators_authorized_keys /grant SYSTEM:`(F`)
83-
icacls C:\ProgramData\ssh\administrators_authorized_keys /grant BUILTIN\Administrators:`(F`)
84-
restart-service sshd
79+
[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String("${AZURE_SSH_PUBLIC_KEY_B64:=''}")) | Add-Content C:/ProgramData/ssh/administrators_authorized_keys
80+
icacls C:/ProgramData/ssh/administrators_authorized_keys /remove "NT AUTHORITY\Authenticated Users"
81+
icacls C:/ProgramData/ssh/administrators_authorized_keys /inheritance:r
82+
icacls C:/ProgramData/ssh/administrators_authorized_keys /grant SYSTEM:F
83+
icacls C:/ProgramData/ssh/administrators_authorized_keys /grant BUILTIN\Administrators:F
84+
restart-service sshd

‎templates/test/cluster-template-prow-machine-pool-windows.yaml‎

Lines changed: 9 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -257,16 +257,16 @@ spec:
257257
# https://github.com/kubernetes-sigs/sig-windows-tools/issues/103#issuecomment-709426828
258258
ipmo C:\k\debug\hns.psm1;
259259
New-HnsNetwork -Type Overlay -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name "External" -AdapterName "Ethernet 2" -SubnetPolicies @(@{Type = "VSID"; VSID = 9999; })
260-
path: C:\create-external-network.ps1
260+
path: C:/create-external-network.ps1
261261
permissions: "0744"
262262
- content: |
263-
[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String("${AZURE_SSH_PUBLIC_KEY_B64:=''}")) > C:\ProgramData\ssh\administrators_authorized_keys
264-
icacls C:\ProgramData\ssh\administrators_authorized_keys /remove "NT AUTHORITY\Authenticated Users"
265-
icacls C:\ProgramData\ssh\administrators_authorized_keys /inheritance:r
266-
icacls C:\ProgramData\ssh\administrators_authorized_keys /grant SYSTEM:`(F`)
267-
icacls C:\ProgramData\ssh\administrators_authorized_keys /grant BUILTIN\Administrators:`(F`)
263+
[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String("${AZURE_SSH_PUBLIC_KEY_B64:=''}")) | Add-Content C:/ProgramData/ssh/administrators_authorized_keys
264+
icacls C:/ProgramData/ssh/administrators_authorized_keys /remove "NT AUTHORITY\Authenticated Users"
265+
icacls C:/ProgramData/ssh/administrators_authorized_keys /inheritance:r
266+
icacls C:/ProgramData/ssh/administrators_authorized_keys /grant SYSTEM:F
267+
icacls C:/ProgramData/ssh/administrators_authorized_keys /grant BUILTIN\Administrators:F
268268
restart-service sshd
269-
path: C:\configure-ssh.ps1
269+
path: C:/configure-ssh.ps1
270270
permissions: "0744"
271271
joinConfiguration:
272272
nodeRegistration:
@@ -276,8 +276,8 @@ spec:
276276
pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:1.4.1
277277
name: '{{ ds.meta_data["local_hostname"] }}'
278278
preKubeadmCommands:
279-
- powershell c:\create-external-network.ps1
280-
- powershell c:\configure-ssh.ps1
279+
- powershell c:/create-external-network.ps1
280+
- powershell c:/configure-ssh.ps1
281281
---
282282
apiVersion: v1
283283
data: ${CNI_RESOURCES_WINDOWS}
@@ -299,24 +299,3 @@ spec:
299299
- kind: ConfigMap
300300
name: cni-${CLUSTER_NAME}-crs-0
301301
strategy: ApplyOnce
302-
---
303-
apiVersion: v1
304-
data: ${WINDOWS_RESOURCES_KUBEPROXY}
305-
kind: ConfigMap
306-
metadata:
307-
name: kubeproxy-${CLUSTER_NAME}-crs-0
308-
namespace: default
309-
---
310-
apiVersion: addons.cluster.x-k8s.io/v1alpha3
311-
kind: ClusterResourceSet
312-
metadata:
313-
name: win-kubeproxy-${CLUSTER_NAME}-crs-0
314-
namespace: default
315-
spec:
316-
clusterSelector:
317-
matchLabels:
318-
cni: ${CLUSTER_NAME}-crs-0
319-
resources:
320-
- kind: ConfigMap
321-
name: kubeproxy-${CLUSTER_NAME}-crs-0
322-
strategy: ApplyOnce

0 commit comments

Comments
 (0)