Skip to content

Commit c0a42ad

Browse files
tintoybrendandburns
tintoy
authored and
brendandburns
committed
Custom validation of server certificate for WebSockets (#103)
* Improve SSL customisation for WebSockets #102 * First test for exec-in-pod over WebSockets. Also, implement basic mock server for testing WebSockets. #102 * Attempt to handle raciness of Watcher tests. #102 * Attempt to handle raciness of ByteBuffer test. #102
1 parent 5b1a831 commit c0a42ad

28 files changed

+2406
-421
lines changed

‎.travis.yml

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,19 @@
11
language: csharp
2-
sudo: false
2+
sudo: false
33
matrix:
44
include:
5-
- dotnet: 2.0.0
6-
mono: none
5+
- mono: none
76
dist: trusty
7+
# We need the .NET Core 2.1 (preview 1) SDK to build. Travis doesn't know how to install this yet.
8+
before_install:
9+
- echo 'Installing .NET Core...'
10+
- export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
11+
- export DOTNET_CLI_TELEMETRY_OPTOUT=1
12+
- curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg
13+
- sudo mv microsoft.gpg /etc/apt/trusted.gpg.d/microsoft.gpg
14+
- sudo sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-trusty-prod trusty main" > /etc/apt/sources.list.d/dotnetdev.list'
15+
- sudo apt-get -qq update
16+
- sudo apt-get install -y dotnet-sdk-2.1.300-preview1-008174
817

918
script:
1019
- ./ci.sh

‎ci.sh

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,12 @@
44
set -e
55

66
# Ensure no compile errors in all projects
7-
find . -name *.csproj -exec dotnet build {} \;
7+
dotnet restore
8+
dotnet build --no-restore
89

910
# Execute Unit tests
1011
cd tests
11-
dotnet restore
12-
dotnet test
12+
dotnet test --no-restore --no-build
13+
if [[ $? != 0 ]]; then
14+
exit 1
15+
fi

‎kubernetes-client.sln

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
21
Microsoft Visual Studio Solution File, Format Version 12.00
32
# Visual Studio 15
43
VisualStudioVersion = 15.0.26430.16
@@ -25,4 +24,7 @@ Global
2524
GlobalSection(SolutionProperties) = preSolution
2625
HideSolutionNode = FALSE
2726
EndGlobalSection
27+
GlobalSection(ExtensibilityGlobals) = postSolution
28+
SolutionGuid = {049A763A-C891-4E8D-80CF-89DD3E22ADC7}
29+
EndGlobalSection
2830
EndGlobal

‎src/CoreFX.cs

Lines changed: 566 additions & 0 deletions
Large diffs are not rendered by default.

‎src/K8sProtocol.cs

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
namespace k8s
2+
{
3+
/// <summary>
4+
/// Well-known WebSocket sub-protocols used by the Kubernetes API.
5+
/// </summary>
6+
public static class K8sProtocol
7+
{
8+
/// <summary>
9+
/// Version 1 of the Kubernetes channel WebSocket protocol.
10+
/// </summary>
11+
/// <remarks>
12+
/// This protocol prepends each binary message with a byte indicating the channel number (zero indexed) that the message was sent on.
13+
/// Messages in both directions should prefix their messages with this channel byte.
14+
///
15+
/// When used for remote execution, the channel numbers are by convention defined to match the POSIX file-descriptors assigned to STDIN, STDOUT, and STDERR (0, 1, and 2).
16+
/// No other conversion is performed on the raw subprotocol - writes are sent as they are received by the server.
17+
///
18+
/// Example client session:
19+
///
20+
/// CONNECT http://server.com with subprotocol "channel.k8s.io"
21+
/// WRITE []byte{0, 102, 111, 111, 10} # send "foo\n" on channel 0 (STDIN)
22+
/// READ []byte{1, 10} # receive "\n" on channel 1 (STDOUT)
23+
/// CLOSE
24+
/// </remarks>
25+
public static readonly string ChannelV1 = "channel.k8s.io";
26+
27+
/// <summary>
28+
/// Version 1 of the Kubernetes Base64-encoded channel WebSocket protocol.
29+
/// </summary>
30+
/// <remarks>
31+
/// This protocol base64 encodes each message with a character representing the channel number (zero indexed) the message was sent on (if the channel number is 1, then the character is '1', i.e. a byte value of 49).
32+
/// Messages in both directions should prefix their messages with this character.
33+
///
34+
/// When used for remote execution, the channel numbers are by convention defined to match the POSIX file-descriptors assigned to STDIN, STDOUT, and STDERR ('0', '1', and '2').
35+
/// The data received on the server is base64 decoded (and must be be valid) and data written by the server to the client is base64 encoded.
36+
///
37+
/// Example client session:
38+
///
39+
/// CONNECT http://server.com with subprotocol "base64.channel.k8s.io"
40+
/// WRITE []byte{48, 90, 109, 57, 118, 67, 103, 111, 61} # send "foo\n" (base64: "Zm9vCgo=") on channel '0' (STDIN)
41+
/// READ []byte{49, 67, 103, 61, 61} # receive "\n" (base64: "Cg==") on channel '1' (STDOUT)
42+
/// CLOSE
43+
/// </remarks>
44+
public static readonly string ChannelBase64V1 = "base64.channel.k8s.io";
45+
}
46+
}

‎src/Kubernetes.WebSocket.cs

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@
22
using Microsoft.Rest;
33
using System;
44
using System.Collections.Generic;
5+
using System.Linq;
56
using System.Net.Http;
67
using System.Net.WebSockets;
8+
using System.Security.Cryptography.X509Certificates;
79
using System.Threading;
810
using System.Threading.Tasks;
911

@@ -204,11 +206,11 @@ public partial class Kubernetes
204206
}
205207
}
206208

207-
// Set Credentials
209+
// Set Credentials
208210
#if NET452
209-
foreach (var cert in ((WebRequestHandler)this.HttpClientHandler).ClientCertificates)
211+
foreach (var cert in ((WebRequestHandler)this.HttpClientHandler).ClientCertificates.OfType<X509Certificate2>())
210212
#else
211-
foreach (var cert in this.HttpClientHandler.ClientCertificates)
213+
foreach (var cert in this.HttpClientHandler.ClientCertificates.OfType<X509Certificate2>())
212214
#endif
213215
{
214216
webSocketBuilder.AddClientCertificate(cert);
@@ -222,6 +224,15 @@ public partial class Kubernetes
222224
webSocketBuilder.SetRequestHeader(_header.Key, string.Join(" ", _header.Value));
223225
}
224226

227+
#if NETCOREAPP2_1
228+
if (this.CaCert != null)
229+
webSocketBuilder.ExpectServerCertificate(this.CaCert);
230+
else
231+
webSocketBuilder.SkipServerCertificateValidation();
232+
233+
webSocketBuilder.Options.RequestedSubProtocols.Add(K8sProtocol.ChannelV1);
234+
#endif // NETCOREAPP2_1
235+
225236
// Send Request
226237
cancellationToken.ThrowIfCancellationRequested();
227238

‎src/KubernetesClient.csproj

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@
99
<PackageProjectUrl>https://github.com/kubernetes-client/csharp</PackageProjectUrl>
1010
<PackageTags>kubernetes;docker;containers;</PackageTags>
1111

12-
<TargetFrameworks>netstandard1.4;net452</TargetFrameworks>
13-
<TargetFrameworks Condition="'$(OS)' != 'Windows_NT'">netstandard1.4</TargetFrameworks>
12+
<TargetFrameworks>netstandard1.4;net452;netcoreapp2.1</TargetFrameworks>
13+
<TargetFrameworks Condition="'$(OS)' != 'Windows_NT'">netstandard1.4;netcoreapp2.1</TargetFrameworks>
1414
<RootNamespace>k8s</RootNamespace>
1515
</PropertyGroup>
1616

‎src/WebSocketBuilder.NetCoreApp2.1.cs

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
#if NETCOREAPP2_1
2+
3+
using System;
4+
using System.Collections.Generic;
5+
using System.Diagnostics;
6+
using System.Net.Security;
7+
using System.Net.WebSockets;
8+
using System.Security.Authentication;
9+
using System.Security.Cryptography.X509Certificates;
10+
using System.Threading;
11+
using System.Threading.Tasks;
12+
13+
namespace k8s
14+
{
15+
/// <summary>
16+
/// The <see cref="WebSocketBuilder"/> creates a new <see cref="WebSocket"/> object which connects to a remote WebSocket.
17+
/// </summary>
18+
public sealed class WebSocketBuilder
19+
{
20+
public KubernetesWebSocketOptions Options { get; } = new KubernetesWebSocketOptions();
21+
22+
public WebSocketBuilder()
23+
{
24+
}
25+
26+
public WebSocketBuilder SetRequestHeader(string headerName, string headerValue)
27+
{
28+
Options.RequestHeaders[headerName] = headerValue;
29+
30+
return this;
31+
}
32+
33+
public WebSocketBuilder AddClientCertificate(X509Certificate2 certificate)
34+
{
35+
Options.ClientCertificates.Add(certificate);
36+
37+
return this;
38+
}
39+
40+
public WebSocketBuilder ExpectServerCertificate(X509Certificate2 serverCertificate)
41+
{
42+
Options.ServerCertificateCustomValidationCallback = (sender, certificate, chain, sslPolicyErrors) =>
43+
{
44+
if (sslPolicyErrors != SslPolicyErrors.RemoteCertificateChainErrors)
45+
return false;
46+
47+
try
48+
{
49+
using (X509Chain certificateChain = new X509Chain())
50+
{
51+
certificateChain.ChainPolicy.ExtraStore.Add(serverCertificate);
52+
certificateChain.ChainPolicy.VerificationFlags = X509VerificationFlags.AllowUnknownCertificateAuthority;
53+
certificateChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck;
54+
55+
return certificateChain.Build(
56+
(X509Certificate2)certificate
57+
);
58+
}
59+
}
60+
catch (Exception chainException)
61+
{
62+
Debug.WriteLine(chainException);
63+
64+
return false;
65+
}
66+
};
67+
68+
return this;
69+
}
70+
71+
public WebSocketBuilder SkipServerCertificateValidation()
72+
{
73+
Options.ServerCertificateCustomValidationCallback = (sender, certificate, chain, sslPolicyErrors) => true;
74+
75+
return this;
76+
}
77+
78+
public async Task<WebSocket> BuildAndConnectAsync(Uri uri, CancellationToken cancellationToken)
79+
{
80+
return await CoreFX.K8sWebSocket.ConnectAsync(uri, Options, cancellationToken).ConfigureAwait(false);
81+
}
82+
}
83+
84+
/// <summary>
85+
/// Options for connecting to Kubernetes web sockets.
86+
/// </summary>
87+
public class KubernetesWebSocketOptions
88+
{
89+
/// <summary>
90+
/// The default size (in bytes) for WebSocket send / receive buffers.
91+
/// </summary>
92+
public static readonly int DefaultBufferSize = 2048;
93+
94+
/// <summary>
95+
/// Create new <see cref="KubernetesWebSocketOptions"/>.
96+
/// </summary>
97+
public KubernetesWebSocketOptions()
98+
{
99+
}
100+
101+
/// <summary>
102+
/// The requested size (in bytes) of the WebSocket send buffer.
103+
/// </summary>
104+
public int SendBufferSize { get; set; } = 2048;
105+
106+
/// <summary>
107+
/// The requested size (in bytes) of the WebSocket receive buffer.
108+
/// </summary>
109+
public int ReceiveBufferSize { get; set; } = 2048;
110+
111+
/// <summary>
112+
/// Custom request headers (if any).
113+
/// </summary>
114+
public Dictionary<string, string> RequestHeaders { get; } = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
115+
116+
/// <summary>
117+
/// Requested sub-protocols (if any).
118+
/// </summary>
119+
public List<string> RequestedSubProtocols { get; } = new List<string>();
120+
121+
/// <summary>
122+
/// Client certificates (if any) to use for authentication.
123+
/// </summary>
124+
public List<X509Certificate2> ClientCertificates = new List<X509Certificate2>();
125+
126+
/// <summary>
127+
/// An optional delegate to use for authenticating the remote server certificate.
128+
/// </summary>
129+
public RemoteCertificateValidationCallback ServerCertificateCustomValidationCallback { get; set; }
130+
131+
/// <summary>
132+
/// An <see cref="SslProtocols"/> value representing the SSL protocols that the client supports.
133+
/// </summary>
134+
public SslProtocols EnabledSslProtocols { get; set; } = SslProtocols.Tls;
135+
136+
/// <summary>
137+
/// The WebSocket keep-alive interval.
138+
/// </summary>
139+
public TimeSpan KeepAliveInterval { get; set; } = TimeSpan.FromSeconds(5);
140+
}
141+
}
142+
143+
#endif // NETCOREAPP2_1

‎src/WebSocketBuilder.cs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
#if !NETCOREAPP2_1
2+
13
using System;
24
using System.Net.WebSockets;
35
using System.Security.Cryptography.X509Certificates;
@@ -19,7 +21,6 @@ public class WebSocketBuilder
1921

2022
public WebSocketBuilder()
2123
{
22-
this.WebSocket = new ClientWebSocket();
2324
}
2425

2526
public virtual WebSocketBuilder SetRequestHeader(string headerName, string headerValue)
@@ -28,7 +29,7 @@ public virtual WebSocketBuilder SetRequestHeader(string headerName, string heade
2829
return this;
2930
}
3031

31-
public virtual WebSocketBuilder AddClientCertificate(X509Certificate certificate)
32+
public virtual WebSocketBuilder AddClientCertificate(X509Certificate2 certificate)
3233
{
3334
this.WebSocket.Options.ClientCertificates.Add(certificate);
3435
return this;
@@ -41,3 +42,5 @@ public virtual async Task<WebSocket> BuildAndConnectAsync(Uri uri, CancellationT
4142
}
4243
}
4344
}
45+
46+
#endif // !NETCOREAPP2_1

‎tests/AuthTests.cs

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,17 @@
1212
using Microsoft.AspNetCore.Server.Kestrel.Https;
1313
using Microsoft.Rest;
1414
using Xunit;
15+
using Xunit.Abstractions;
1516

1617
namespace k8s.Tests
1718
{
18-
public class AuthTests
19-
{
19+
public class AuthTests
20+
: TestBase
21+
{
22+
public AuthTests(ITestOutputHelper testOutput) : base(testOutput)
23+
{
24+
}
25+
2026
private static HttpOperationResponse<V1PodList> ExecuteListPods(IKubernetes client)
2127
{
2228
return client.ListNamespacedPodWithHttpMessagesAsync("default").Result;
@@ -25,7 +31,7 @@ private static HttpOperationResponse<V1PodList> ExecuteListPods(IKubernetes clie
2531
[Fact]
2632
public void Anonymous()
2733
{
28-
using (var server = new MockKubeApiServer())
34+
using (var server = new MockKubeApiServer(TestOutput))
2935
{
3036
var client = new Kubernetes(new KubernetesClientConfiguration
3137
{
@@ -38,7 +44,7 @@ public void Anonymous()
3844
Assert.Equal(1, listTask.Body.Items.Count);
3945
}
4046

41-
using (var server = new MockKubeApiServer(cxt =>
47+
using (var server = new MockKubeApiServer(TestOutput, cxt =>
4248
{
4349
cxt.Response.StatusCode = (int) HttpStatusCode.Unauthorized;
4450
return Task.FromResult(false);
@@ -61,7 +67,7 @@ public void BasicAuth()
6167
const string testName = "test_name";
6268
const string testPassword = "test_password";
6369

64-
using (var server = new MockKubeApiServer(cxt =>
70+
using (var server = new MockKubeApiServer(TestOutput, cxt =>
6571
{
6672
var header = cxt.Request.Headers["Authorization"].FirstOrDefault();
6773

@@ -167,7 +173,7 @@ public void Cert()
167173

168174
var clientCertificateValidationCalled = false;
169175

170-
using (var server = new MockKubeApiServer(listenConfigure: options =>
176+
using (var server = new MockKubeApiServer(TestOutput, listenConfigure: options =>
171177
{
172178
options.UseHttps(new HttpsConnectionAdapterOptions
173179
{
@@ -249,7 +255,7 @@ public void Token()
249255
{
250256
const string token = "testingtoken";
251257

252-
using (var server = new MockKubeApiServer(cxt =>
258+
using (var server = new MockKubeApiServer(TestOutput, cxt =>
253259
{
254260
var header = cxt.Request.Headers["Authorization"].FirstOrDefault();
255261

0 commit comments

Comments
 (0)