Skip to content

Commit f8f9cce

Browse files
buehlerbuehler
committed
feat(startup): add operator builder and refactor the startup code. (#47)
BREAKING CHANGE: the default way the operator was started was removed. Please change to the new "generic host" way of starting the operator. In the Program.cs file, the operator wrapper is still needed for the additional operator commands. BREAKING CHANGE: Complete overhaul of the KubernetesOperator class The migration path will be to create a new asp.net project and then add the kubernetes operator back again. BREAKING CHANGE: Removed the Kubernetes Operator class and inserted generic host extensions for the CreateHostBuilder method of asp.netcore. BREAKING CHANGE: Removed the kubernetes test operator. Instead, a web application factory (called KubernetesOperatorFactory) was added to be used by the tests to startup the server and configure the test application further. BREAKING CHANGE: Removed the AddResourceFinalizer extension method. This part is now found in the resource controller. There the entity types are clear and a finalizer can be attached with the client.
1 parent aa14f62 commit f8f9cce

37 files changed

+585
-741
lines changed

‎DotnetOperatorSdk.sln‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "KubeOps.TestOperator", "tes
2222
EndProject
2323
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "KubeOps.TestOperator.Test", "tests\KubeOps.TestOperator.Test\KubeOps.TestOperator.Test.csproj", "{B374D7E4-E9BA-47F8-B1A4-440DECD376E4}"
2424
EndProject
25-
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KubeOps.GenericTest", "tests\KubeOps.GenericTest\KubeOps.GenericTest.csproj", "{714D8DD8-AF4B-4990-AE63-0258BC3E3CD3}"
26-
EndProject
2725
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "config", "config", "{EF94C48A-1D2A-459B-B397-A130463FF52D}"
2826
ProjectSection(SolutionItems) = preProject
2927
config\CodeAnalysis.targets = config\CodeAnalysis.targets
@@ -58,10 +56,6 @@ Global
5856
{B374D7E4-E9BA-47F8-B1A4-440DECD376E4}.Debug|Any CPU.Build.0 = Debug|Any CPU
5957
{B374D7E4-E9BA-47F8-B1A4-440DECD376E4}.Release|Any CPU.ActiveCfg = Release|Any CPU
6058
{B374D7E4-E9BA-47F8-B1A4-440DECD376E4}.Release|Any CPU.Build.0 = Release|Any CPU
61-
{714D8DD8-AF4B-4990-AE63-0258BC3E3CD3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
62-
{714D8DD8-AF4B-4990-AE63-0258BC3E3CD3}.Debug|Any CPU.Build.0 = Debug|Any CPU
63-
{714D8DD8-AF4B-4990-AE63-0258BC3E3CD3}.Release|Any CPU.ActiveCfg = Release|Any CPU
64-
{714D8DD8-AF4B-4990-AE63-0258BC3E3CD3}.Release|Any CPU.Build.0 = Release|Any CPU
6559
EndGlobalSection
6660
GlobalSection(SolutionProperties) = preSolution
6761
HideSolutionNode = FALSE
@@ -72,7 +66,6 @@ Global
7266
{D47717CB-A02E-4B12-BAA8-1D7F8BAE9BBD} = {9AF95FE4-DA1F-4BB0-B60E-23FCFA6AAAA2}
7367
{751BDC14-D75F-4DDE-9C45-2432041FBCAC} = {50E9B964-68F7-4B9F-BEA8-165CE45BC5C6}
7468
{B374D7E4-E9BA-47F8-B1A4-440DECD376E4} = {50E9B964-68F7-4B9F-BEA8-165CE45BC5C6}
75-
{714D8DD8-AF4B-4990-AE63-0258BC3E3CD3} = {50E9B964-68F7-4B9F-BEA8-165CE45BC5C6}
7669
EndGlobalSection
7770
GlobalSection(ExtensibilityGlobals) = postSolution
7871
SolutionGuid = {ADAA5DFB-A7E5-490C-88EF-48D67C715F24}

‎src/KubeOps/Build/Program.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,6 @@ namespace KubeOps.Build
55
{
66
internal static class Program
77
{
8-
public static Task Main(string[] args) => new KubernetesOperator().Run(args);
8+
public static Task Main(string[] _) => Task.CompletedTask;
99
}
1010
}

‎src/KubeOps/KubeOps.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
<PackageReference Include="KubernetesClient" Version="2.0.29" />
2626
<PackageReference Include="McMaster.Extensions.CommandLineUtils" Version="3.0.0" />
2727
<PackageReference Include="McMaster.Extensions.Hosting.CommandLine" Version="3.0.0" />
28+
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="3.1.7" />
2829
<PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="3.1.7" />
2930
<PackageReference Include="Namotion.Reflection" Version="1.0.12" />
3031
<PackageReference Include="prometheus-net.AspNetCore" Version="3.6.0" />

‎src/KubeOps/Operator/OperatorStartup.cs‎ renamed to ‎src/KubeOps/Operator/ApplicationBuilderExtensions.cs‎

Lines changed: 8 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,28 @@
1-
using KubeOps.Operator.DevOps;
1+
using KubeOps.Operator.Builder;
22
using Microsoft.AspNetCore.Builder;
33
using Microsoft.AspNetCore.Diagnostics.HealthChecks;
44
using Microsoft.Extensions.DependencyInjection;
55
using Prometheus;
66

77
namespace KubeOps.Operator
88
{
9-
public class OperatorStartup
9+
public static class ApplicationBuilderExtensions
1010
{
11-
internal const string LivenessTag = "liveness";
12-
internal const string ReadinessTag = "readiness";
13-
14-
public void ConfigureServices(IServiceCollection services)
15-
{
16-
services
17-
.AddHealthChecks()
18-
.ForwardToPrometheus();
19-
20-
services.AddHealthCheck<ControllerLivenessCheck>();
21-
}
22-
23-
public void Configure(IApplicationBuilder app, OperatorSettings settings)
11+
public static void UseKubernetesOperator(
12+
this IApplicationBuilder app)
2413
{
2514
app.UseRouting();
2615
app.UseEndpoints(
2716
endpoints =>
2817
{
18+
var settings = app.ApplicationServices.GetRequiredService<OperatorSettings>();
19+
2920
endpoints.MapHealthChecks(
3021
settings.LivenessEndpoint,
31-
new HealthCheckOptions { Predicate = reg => reg.Tags.Contains(LivenessTag) });
22+
new HealthCheckOptions { Predicate = reg => reg.Tags.Contains(OperatorBuilder.LivenessTag) });
3223
endpoints.MapHealthChecks(
3324
settings.ReadinessEndpoint,
34-
new HealthCheckOptions { Predicate = reg => reg.Tags.Contains(ReadinessTag) });
25+
new HealthCheckOptions { Predicate = reg => reg.Tags.Contains(OperatorBuilder.ReadinessTag) });
3526

3627
endpoints.MapMetrics(settings.MetricsEndpoint);
3728
});

‎src/KubeOps/Operator/Builder/IOperatorBuilder.cs‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
using k8s;
2+
using k8s.Models;
3+
using KubeOps.Operator.Controller;
4+
using KubeOps.Operator.Finalizer;
5+
using Microsoft.Extensions.DependencyInjection;
6+
using Microsoft.Extensions.Diagnostics.HealthChecks;
7+
8+
namespace KubeOps.Operator.Builder
9+
{
10+
/// <summary>
11+
/// Builder for specific services of the kubernetes operator.
12+
/// </summary>
13+
public interface IOperatorBuilder
14+
{
15+
/// <summary>
16+
/// Returns the original service collection.
17+
/// </summary>
18+
IServiceCollection Services { get; }
19+
20+
/// <summary>
21+
/// Adds an <see cref="IHealthCheck"/> to "both" healthy probes.
22+
/// The health check will be executed on the "/health" and "/ready" route.
23+
/// (The routes can be configured via operator settings).
24+
/// </summary>
25+
/// <param name="name">An optional name of the health check.</param>
26+
/// <typeparam name="THealthCheck">The type that should be added to both probes.</typeparam>
27+
/// <returns>The builder for chaining.</returns>
28+
IOperatorBuilder AddHealthCheck<THealthCheck>(string? name = default)
29+
where THealthCheck : class, IHealthCheck;
30+
31+
/// <summary>
32+
/// Adds an <see cref="IHealthCheck"/> to the readiness probe.
33+
/// The health check will be executed on the "/ready" route.
34+
/// (The route can be configured via operator settings).
35+
/// </summary>
36+
/// <param name="name">An optional name of the readiness check.</param>
37+
/// <typeparam name="TReadinessCheck">The type that should be added to the probe.</typeparam>
38+
/// <returns>The builder for chaining.</returns>
39+
IOperatorBuilder AddReadinessCheck<TReadinessCheck>(string? name = default)
40+
where TReadinessCheck : class, IHealthCheck;
41+
42+
/// <summary>
43+
/// Adds an <see cref="IHealthCheck"/> to the liveness probe.
44+
/// The health check will be executed on the "/health" route.
45+
/// (The route can be configured via operator settings).
46+
/// </summary>
47+
/// <param name="name">An optional name of the liveness check.</param>
48+
/// <typeparam name="TLivenessCheck">The type that should be added to the probe.</typeparam>
49+
/// <returns>The builder for chaining.</returns>
50+
IOperatorBuilder AddLivenessCheck<TLivenessCheck>(string? name = default)
51+
where TLivenessCheck : class, IHealthCheck;
52+
53+
/// <summary>
54+
/// Adds a resource controller for an entity of <see cref="IKubernetesObject{TMetadata}"/>.
55+
/// This controller is taking care for the reconciliation of that particular entity.
56+
/// </summary>
57+
/// <typeparam name="TController">The type of the controller to add.</typeparam>
58+
/// <returns>The builder for chaining.</returns>
59+
IOperatorBuilder AddController<TController>()
60+
where TController : class, IResourceController;
61+
62+
/// <summary>
63+
/// Adds a resource finalizer. Finalizers take care of "to be deleted" instances of
64+
/// an entity that needs to asynchronously perfrom some tasks.
65+
/// </summary>
66+
/// <typeparam name="TFinalizer">The type of the finalizer to add.</typeparam>
67+
/// <returns>The builder for chaining.</returns>
68+
IOperatorBuilder AddFinalizer<TFinalizer>()
69+
where TFinalizer : class, IResourceFinalizer;
70+
}
71+
}

‎src/KubeOps/Operator/Builder/OperatorBuilder.cs‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using k8s;
4+
using KubeOps.Operator.Caching;
5+
using KubeOps.Operator.Client;
6+
using KubeOps.Operator.Controller;
7+
using KubeOps.Operator.DevOps;
8+
using KubeOps.Operator.Finalizer;
9+
using KubeOps.Operator.Queue;
10+
using KubeOps.Operator.Serialization;
11+
using KubeOps.Operator.Watcher;
12+
using Microsoft.Extensions.DependencyInjection;
13+
using Microsoft.Extensions.Diagnostics.HealthChecks;
14+
using Newtonsoft.Json;
15+
using Newtonsoft.Json.Converters;
16+
using Newtonsoft.Json.Serialization;
17+
using Prometheus;
18+
using YamlDotNet.Serialization;
19+
20+
namespace KubeOps.Operator.Builder
21+
{
22+
internal class OperatorBuilder : IOperatorBuilder
23+
{
24+
internal const string LivenessTag = "liveness";
25+
internal const string ReadinessTag = "readiness";
26+
27+
public OperatorBuilder(IServiceCollection services)
28+
{
29+
Services = services;
30+
}
31+
32+
public IServiceCollection Services { get; }
33+
34+
public IOperatorBuilder AddHealthCheck<THealthCheck>(string? name = default)
35+
where THealthCheck : class, IHealthCheck
36+
{
37+
Services
38+
.AddHealthChecks()
39+
.AddCheck<THealthCheck>(
40+
name ?? typeof(THealthCheck).Name,
41+
tags: new[] { ReadinessTag, LivenessTag });
42+
43+
return this;
44+
}
45+
46+
public IOperatorBuilder AddReadinessCheck<TReadinessCheck>(string? name = default)
47+
where TReadinessCheck : class, IHealthCheck
48+
{
49+
Services
50+
.AddHealthChecks()
51+
.AddCheck<TReadinessCheck>(
52+
name ?? typeof(TReadinessCheck).Name,
53+
tags: new[] { ReadinessTag });
54+
55+
return this;
56+
}
57+
58+
public IOperatorBuilder AddLivenessCheck<TLivenessCheck>(string? name = default)
59+
where TLivenessCheck : class, IHealthCheck
60+
{
61+
Services
62+
.AddHealthChecks()
63+
.AddCheck<TLivenessCheck>(
64+
name ?? typeof(TLivenessCheck).Name,
65+
tags: new[] { LivenessTag });
66+
67+
return this;
68+
}
69+
70+
public IOperatorBuilder AddController<TController>()
71+
where TController : class, IResourceController
72+
{
73+
Services.AddHostedService<TController>();
74+
75+
return this;
76+
}
77+
78+
public IOperatorBuilder AddFinalizer<TFinalizer>()
79+
where TFinalizer : class, IResourceFinalizer
80+
{
81+
Services.AddTransient(typeof(IResourceFinalizer), typeof(TFinalizer));
82+
83+
return this;
84+
}
85+
86+
internal IOperatorBuilder AddOperatorBase(OperatorSettings settings)
87+
{
88+
Services.AddSingleton(settings);
89+
90+
// support lazy service resolution
91+
Services.AddTransient(typeof(Lazy<>), typeof(LazyService<>));
92+
93+
Services.AddTransient(
94+
_ => new JsonSerializerSettings
95+
{
96+
ContractResolver = new NamingConvention(),
97+
Converters = new List<JsonConverter>
98+
{
99+
new StringEnumConverter { NamingStrategy = new CamelCaseNamingStrategy() },
100+
},
101+
});
102+
JsonConvert.DefaultSettings = () => new JsonSerializerSettings
103+
{
104+
ContractResolver = new NamingConvention(),
105+
Converters = new List<JsonConverter>
106+
{
107+
new StringEnumConverter { NamingStrategy = new CamelCaseNamingStrategy() },
108+
},
109+
};
110+
111+
Services.AddTransient(
112+
_ => new SerializerBuilder()
113+
.ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitNull)
114+
.WithNamingConvention(new NamingConvention())
115+
.Build());
116+
117+
Services.AddTransient<EntitySerializer>();
118+
119+
Services.AddTransient<IKubernetesClient, KubernetesClient>();
120+
Services.AddSingleton<IKubernetes>(
121+
_ =>
122+
{
123+
var config = KubernetesClientConfiguration.BuildDefaultConfig();
124+
125+
return new Kubernetes(config, new ClientUrlFixer())
126+
{
127+
SerializationSettings =
128+
{
129+
ContractResolver = new NamingConvention(),
130+
Converters = new List<JsonConverter>
131+
{ new StringEnumConverter { NamingStrategy = new CamelCaseNamingStrategy() } },
132+
},
133+
DeserializationSettings =
134+
{
135+
ContractResolver = new NamingConvention(),
136+
Converters = new List<JsonConverter>
137+
{ new StringEnumConverter { NamingStrategy = new CamelCaseNamingStrategy() } },
138+
},
139+
};
140+
});
141+
142+
Services.AddTransient(typeof(IResourceCache<>), typeof(ResourceCache<>));
143+
Services.AddTransient(typeof(IResourceWatcher<>), typeof(ResourceWatcher<>));
144+
Services.AddTransient(typeof(IResourceEventQueue<>), typeof(ResourceEventQueue<>));
145+
Services.AddTransient(typeof(IResourceServices<>), typeof(ResourceServices<>));
146+
147+
// Support for healthchecks and prometheus.
148+
Services
149+
.AddHealthChecks()
150+
.ForwardToPrometheus();
151+
152+
// Add the default controller liveness check.
153+
AddHealthCheck<ControllerLivenessCheck>();
154+
155+
return this;
156+
}
157+
}
158+
}

0 commit comments

Comments
 (0)