Skip to content

Commit 57ca6f0

Browse files
committed
Enable OpenSSH on Windows nodes in test clusters.
Also switches to the most recent 64-bit version of OpenSSH for Windows. Tested: PROJECT=${CLOUDSDK_CORE_PROJECT} KUBERNETES_SKIP_CONFIRM=y NUM_NODES=2 \ NUM_WINDOWS_NODES=2 KUBE_GCE_ENABLE_IP_ALIASES=true TEST_CLUSTER=true \ ./cluster/kube-up.sh
1 parent f7a6b0a commit 57ca6f0

File tree

4 files changed

+293
-1
lines changed

4 files changed

+293
-1
lines changed

cluster/gce/windows/configure.ps1

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,18 @@ function FetchAndImport-ModuleFromMetadata {
7777
Import-Module -Force C:\$Filename
7878
}
7979

80+
# Returns true if this node is part of a test cluster (see
81+
# cluster/gce/config-test.sh).
82+
#
83+
# $kube_env must be set before calling this function.
84+
function Test-IsTestCluster {
85+
if ($kube_env.Contains('TEST_CLUSTER') -and `
86+
($kube_env['TEST_CLUSTER'] -eq 'true')) {
87+
return $true
88+
}
89+
return $false
90+
}
91+
8092
try {
8193
# Don't use FetchAndImport-ModuleFromMetadata for common.psm1 - the common
8294
# module includes variables and functions that any other function may depend
@@ -92,6 +104,14 @@ try {
92104

93105
Set-PrerequisiteOptions
94106
$kube_env = Fetch-KubeEnv
107+
108+
if (Test-IsTestCluster) {
109+
Log-Output 'Test cluster detected, installing OpenSSH.'
110+
FetchAndImport-ModuleFromMetadata 'install-ssh-psm1' 'install-ssh.psm1'
111+
InstallAndStart-OpenSsh
112+
StartProcess-WriteSshKeys
113+
}
114+
95115
Set-EnvironmentVars
96116
Create-Directories
97117
Download-HelperScripts

cluster/gce/windows/node-helper.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ function get-windows-node-instance-metadata-from-file {
2727
metadata+="windows-startup-script-ps1=${KUBE_ROOT}/cluster/gce/windows/configure.ps1,"
2828
metadata+="common-psm1=${KUBE_ROOT}/cluster/gce/windows/common.psm1,"
2929
metadata+="k8s-node-setup-psm1=${KUBE_ROOT}/cluster/gce/windows/k8s-node-setup.psm1,"
30-
metadata+="user-profile-psm1=${KUBE_ROOT}/cluster/gce/windows/user-profile.psm1,"
30+
metadata+="install-ssh-psm1=${KUBE_ROOT}/cluster/gce/windows/testonly/install-ssh.psm1,"
31+
metadata+="user-profile-psm1=${KUBE_ROOT}/cluster/gce/windows/testonly/user-profile.psm1,"
3132
metadata+="${NODE_EXTRA_METADATA}"
3233
echo "${metadata}"
3334
}
Lines changed: 271 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,271 @@
1+
# Copyright 2019 The Kubernetes Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
<#
16+
.SYNOPSIS
17+
Library for installing and running Win64-OpenSSH. NOT FOR PRODUCTION USE.
18+
19+
.NOTES
20+
This module depends on common.psm1. This module depends on third-party code
21+
which has not been security-reviewed, so it should only be used for test
22+
clusters. DO NOT USE THIS MODULE FOR PRODUCTION.
23+
#>
24+
25+
Import-Module -Force C:\common.psm1
26+
27+
$OPENSSH_ROOT = 'C:\Program Files\OpenSSH'
28+
$USER_PROFILE_MODULE = 'C:\user-profile.psm1'
29+
$WRITE_SSH_KEYS_SCRIPT = 'C:\write-ssh-keys.ps1'
30+
31+
# Starts the Win64-OpenSSH services and configures them to automatically start
32+
# on subsequent boots.
33+
function Start_OpenSshServices {
34+
ForEach ($service in ("sshd", "ssh-agent")) {
35+
net start ${service}
36+
Set-Service ${service} -StartupType Automatic
37+
}
38+
}
39+
40+
# Installs open-ssh using the instructions in
41+
# https://github.com/PowerShell/Win32-OpenSSH/wiki/Install-Win32-OpenSSH.
42+
#
43+
# After installation run StartProcess-WriteSshKeys to fetch ssh keys from the
44+
# metadata server.
45+
function InstallAndStart-OpenSsh {
46+
if (-not (ShouldWrite-File $OPENSSH_ROOT)) {
47+
Log-Output "Starting already-installed OpenSSH services"
48+
Start_OpenSshServices
49+
return
50+
}
51+
elseif (Test-Path $OPENSSH_ROOT) {
52+
Log-Output ("OpenSSH directory already exists, attempting to run its " +
53+
"uninstaller before reinstalling")
54+
powershell.exe `
55+
-ExecutionPolicy Bypass `
56+
-File "$OPENSSH_ROOT\OpenSSH-Win64\uninstall-sshd.ps1"
57+
rm -Force -Recurse $OPENSSH_ROOT\OpenSSH-Win64
58+
}
59+
60+
# Download open-ssh.
61+
# Use TLS 1.2: needed for Invoke-WebRequest downloads from github.com.
62+
[Net.ServicePointManager]::SecurityProtocol = `
63+
[Net.SecurityProtocolType]::Tls12
64+
$url = ("https://github.com/PowerShell/Win32-OpenSSH/releases/download/" +
65+
"v7.9.0.0p1-Beta/OpenSSH-Win64.zip")
66+
$ProgressPreference = 'SilentlyContinue'
67+
Invoke-WebRequest $url -OutFile C:\openssh-win64.zip
68+
69+
# Unzip and install open-ssh
70+
Expand-Archive -Force C:\openssh-win64.zip -DestinationPath $OPENSSH_ROOT
71+
powershell.exe `
72+
-ExecutionPolicy Bypass `
73+
-File "$OPENSSH_ROOT\OpenSSH-Win64\install-sshd.ps1"
74+
75+
# Disable password-based authentication.
76+
$sshd_config_default = "$OPENSSH_ROOT\OpenSSH-Win64\sshd_config_default"
77+
$sshd_config = 'C:\ProgramData\ssh\sshd_config'
78+
New-Item -Force -ItemType Directory -Path "C:\ProgramData\ssh\" | Out-Null
79+
# SSH config files must be UTF-8 encoded:
80+
# https://github.com/PowerShell/Win32-OpenSSH/issues/862
81+
# https://github.com/PowerShell/Win32-OpenSSH/wiki/Various-Considerations
82+
(Get-Content $sshd_config_default).`
83+
replace('#PasswordAuthentication yes', 'PasswordAuthentication no') |
84+
Set-Content -Encoding UTF8 $sshd_config
85+
86+
# Configure the firewall to allow inbound SSH connections
87+
if (Get-NetFirewallRule -ErrorAction SilentlyContinue sshd) {
88+
Get-NetFirewallRule sshd | Remove-NetFirewallRule
89+
}
90+
New-NetFirewallRule `
91+
-Name sshd `
92+
-DisplayName 'OpenSSH Server (sshd)' `
93+
-Enabled True `
94+
-Direction Inbound `
95+
-Protocol TCP `
96+
-Action Allow `
97+
-LocalPort 22
98+
99+
Start_OpenSshServices
100+
}
101+
102+
function Setup_WriteSshKeysScript {
103+
if (-not (ShouldWrite-File $WRITE_SSH_KEYS_SCRIPT)) {
104+
return
105+
}
106+
107+
# Fetch helper module for manipulating Windows user profiles.
108+
if (ShouldWrite-File $USER_PROFILE_MODULE) {
109+
$module = Get-InstanceMetadataValue 'user-profile-psm1'
110+
New-Item -ItemType file -Force $USER_PROFILE_MODULE | Out-Null
111+
Set-Content $USER_PROFILE_MODULE $module
112+
}
113+
114+
# TODO(pjh): check if we still need to write authorized_keys to users-specific
115+
# directories, or if just writing to the centralized keys file for
116+
# Administrators on the system is sufficient (does our log-dump user have
117+
# Administrator rights?).
118+
New-Item -Force -ItemType file ${WRITE_SSH_KEYS_SCRIPT} | Out-Null
119+
Set-Content ${WRITE_SSH_KEYS_SCRIPT} `
120+
'Import-Module -Force USER_PROFILE_MODULE
121+
# For [System.Web.Security.Membership]::GeneratePassword():
122+
Add-Type -AssemblyName System.Web
123+
124+
$poll_interval = 10
125+
126+
while($true) {
127+
$r1 = ""
128+
$r2 = ""
129+
# Try both the new "ssh-keys" and the legacy "sshSkeys" attributes for
130+
# compatibility. The Invoke-RestMethods calls will fail when these attributes
131+
# do not exist, or they may fail when the connection to the metadata server
132+
# gets disrupted while we set up container networking on the node.
133+
try {
134+
$r1 = Invoke-RestMethod -Headers @{"Metadata-Flavor"="Google"} -Uri `
135+
"http://metadata.google.internal/computeMetadata/v1/project/attributes/ssh-keys"
136+
} catch {}
137+
try {
138+
$r2 = Invoke-RestMethod -Headers @{"Metadata-Flavor"="Google"} -Uri `
139+
"http://metadata.google.internal/computeMetadata/v1/project/attributes/sshKeys"
140+
} catch {}
141+
$response= $r1 + $r2
142+
143+
# Split the response into lines; handle both \r\n and \n line breaks.
144+
$tuples = $response -split "\r?\n"
145+
146+
$users_to_keys = @{}
147+
foreach($line in $tuples) {
148+
if ([string]::IsNullOrEmpty($line)) {
149+
continue
150+
}
151+
# The final parameter to -Split is the max number of strings to return, so
152+
# this only splits on the first colon.
153+
$username, $key = $line -Split ":",2
154+
155+
# Detect and skip keys without associated usernames, which may come back
156+
# from the legacy sshKeys metadata.
157+
if (($username -like "ssh-*") -or ($username -like "ecdsa-*")) {
158+
Write-Error "Skipping key without username: $username"
159+
continue
160+
}
161+
if (-not $users_to_keys.ContainsKey($username)) {
162+
$users_to_keys[$username] = @($key)
163+
}
164+
else {
165+
$keyList = $users_to_keys[$username]
166+
$users_to_keys[$username] = $keyList + $key
167+
}
168+
}
169+
$users_to_keys.GetEnumerator() | ForEach-Object {
170+
$username = $_.key
171+
172+
# We want to create an authorized_keys file in the user profile directory
173+
# for each user, but if we create the directory before that user profile
174+
# has been created first by Windows, then Windows will create a different
175+
# user profile directory that looks like "<user>.KUBERNETES-MINI" and sshd
176+
# will look for the authorized_keys file in THAT directory. In other words,
177+
# we need to create the user first before we can put the authorized_keys
178+
# file in that user profile directory. The user-profile.psm1 module (NOT
179+
# FOR PRODUCTION USE!) has Create-NewProfile which achieves this.
180+
#
181+
# Run "Get-Command -Module Microsoft.PowerShell.LocalAccounts" to see the
182+
# build-in commands for users and groups. For some reason the New-LocalUser
183+
# command does not create the user profile directory, so we use the
184+
# auxiliary user-profile.psm1 instead.
185+
186+
$pw = [System.Web.Security.Membership]::GeneratePassword(16,2)
187+
try {
188+
# Create-NewProfile will throw this when the user profile already exists:
189+
# Create-NewProfile : Exception calling "SetInfo" with "0" argument(s):
190+
# "The account already exists."
191+
# Just catch it and ignore it.
192+
Create-NewProfile $username $pw -ErrorAction Stop
193+
194+
# Add the user to the Administrators group, otherwise we will not have
195+
# privilege when we ssh.
196+
Add-LocalGroupMember -Group Administrators -Member $username
197+
} catch {}
198+
199+
$user_dir = "C:\Users\" + $username
200+
if (-not (Test-Path $user_dir)) {
201+
# If for some reason Create-NewProfile failed to create the user profile
202+
# directory just continue on to the next user.
203+
continue
204+
}
205+
206+
# NOTE: there is a race condition here where someone could try to ssh to
207+
# this node in-between when we clear out the authorized_keys file and when
208+
# we write keys to it. Oh well.
209+
$user_keys_file = -join($user_dir, "\.ssh\authorized_keys")
210+
New-Item -ItemType file -Force $user_keys_file | Out-Null
211+
212+
# New for v7.9.0.0: administrators_authorized_keys file. For permission
213+
# information see
214+
# https://github.com/PowerShell/Win32-OpenSSH/wiki/Security-protection-of-various-files-in-Win32-OpenSSH#administrators_authorized_keys.
215+
$administrator_keys_file = ${env:ProgramData} + `
216+
"\ssh\administrators_authorized_keys"
217+
New-Item -ItemType file -Force $administrator_keys_file | Out-Null
218+
icacls $administrator_keys_file /inheritance:r | Out-Null
219+
icacls $administrator_keys_file /grant SYSTEM:`(F`) | Out-Null
220+
icacls $administrator_keys_file /grant BUILTIN\Administrators:`(F`) | `
221+
Out-Null
222+
223+
ForEach ($ssh_key in $_.value) {
224+
# authorized_keys and other ssh config files must be UTF-8 encoded:
225+
# https://github.com/PowerShell/Win32-OpenSSH/issues/862
226+
# https://github.com/PowerShell/Win32-OpenSSH/wiki/Various-Considerations
227+
Add-Content -Encoding UTF8 $user_keys_file $ssh_key
228+
Add-Content -Encoding UTF8 $administrator_keys_file $ssh_key
229+
}
230+
}
231+
Start-Sleep -sec $poll_interval
232+
}'.replace('USER_PROFILE_MODULE', $USER_PROFILE_MODULE)
233+
Log-Output ("${WRITE_SSH_KEYS_SCRIPT}:`n" +
234+
"$(Get-Content -Raw ${WRITE_SSH_KEYS_SCRIPT})")
235+
}
236+
237+
# Starts a background process that retrieves ssh keys from the metadata server
238+
# and writes them to user-specific directories. Intended for use only by test
239+
# clusters!!
240+
#
241+
# While this is running it should be possible to SSH to the Windows node using:
242+
# gcloud compute ssh <username>@<instance> --zone=<zone>
243+
# or:
244+
# ssh -i ~/.ssh/google_compute_engine -o 'IdentitiesOnly yes' \
245+
# <username>@<instance_external_ip>
246+
# or copy files using:
247+
# gcloud compute scp <username>@<instance>:C:\\path\\to\\file.txt \
248+
# path/to/destination/ --zone=<zone>
249+
#
250+
# If the username you're using does not already have a project-level SSH key
251+
# (run "gcloud compute project-info describe --flatten
252+
# commonInstanceMetadata.items.ssh-keys" to check), run gcloud compute ssh with
253+
# that username once to add a new project-level SSH key, wait one minute for
254+
# StartProcess-WriteSshKeys to pick it up, then try to ssh/scp again.
255+
function StartProcess-WriteSshKeys {
256+
Setup_WriteSshKeysScript
257+
258+
# TODO(pjh): check if such a process is already running before starting
259+
# another one.
260+
$write_keys_process = Start-Process `
261+
-FilePath "powershell.exe" `
262+
-ArgumentList @("-Command", ${WRITE_SSH_KEYS_SCRIPT}) `
263+
-WindowStyle Hidden -PassThru `
264+
-RedirectStandardOutput "NUL" `
265+
-RedirectStandardError C:\write-ssh-keys.err
266+
Log-Output "Started background process to write SSH keys"
267+
Log-Output "$(${write_keys_process} | Out-String)"
268+
}
269+
270+
# Export all public functions:
271+
Export-ModuleMember -Function *-*
File renamed without changes.

0 commit comments

Comments
 (0)