Skip to content

Commit 3d4df6a

Browse files
simonpasquiersimonpasquier
committed
feat: use TLS profile to configure CMO server
1 parent 61407b0 commit 3d4df6a

File tree

4 files changed

+109
-79
lines changed

4 files changed

+109
-79
lines changed

‎cmd/operator/main.go‎

Lines changed: 71 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,12 +25,18 @@ import (
2525
"strings"
2626
"syscall"
2727

28+
configv1client "github.com/openshift/client-go/config/clientset/versioned"
29+
"github.com/openshift/library-go/pkg/operator/events"
2830
"golang.org/x/sync/errgroup"
2931
"k8s.io/apimachinery/pkg/util/yaml"
32+
"k8s.io/client-go/kubernetes"
33+
"k8s.io/client-go/rest"
3034
"k8s.io/client-go/tools/clientcmd"
3135
"k8s.io/klog/v2"
36+
"k8s.io/utils/clock"
3237
runtimelog "sigs.k8s.io/controller-runtime/pkg/log"
3338

39+
"github.com/openshift/cluster-monitoring-operator/pkg/client"
3440
"github.com/openshift/cluster-monitoring-operator/pkg/manifests"
3541
cmo "github.com/openshift/cluster-monitoring-operator/pkg/operator"
3642
"github.com/openshift/cluster-monitoring-operator/pkg/server"
@@ -83,6 +89,42 @@ type telemetryConfig struct {
8389
Matches []string `json:"matches"`
8490
}
8591

92+
func newClient(
93+
ctx context.Context,
94+
config *rest.Config,
95+
version string,
96+
namespace, namespaceUserWorkload string,
97+
) (*client.Client, error) {
98+
kclient, err := kubernetes.NewForConfig(config)
99+
if err != nil {
100+
return nil, fmt.Errorf("creating kubernetes clientset client: %w", err)
101+
}
102+
controllerRef, err := events.GetControllerReferenceForCurrentPod(ctx, kclient, namespace, nil)
103+
if err != nil {
104+
klog.Warningf("unable to get owner reference (falling back to namespace): %v", err)
105+
}
106+
107+
eventRecorder := events.NewKubeRecorderWithOptions(
108+
kclient.CoreV1().Events(namespace),
109+
events.RecommendedClusterSingletonCorrelatorOptions(),
110+
"cluster-monitoring-operator",
111+
controllerRef,
112+
clock.RealClock{},
113+
)
114+
115+
configClient, err := configv1client.NewForConfig(config)
116+
if err != nil {
117+
return nil, err
118+
}
119+
120+
c, err := client.NewForConfig(config, version, namespace, namespaceUserWorkload, client.KubernetesClient(kclient), client.OpenshiftConfigClient(configClient), client.EventRecorder(eventRecorder))
121+
if err != nil {
122+
return nil, err
123+
}
124+
125+
return c, nil
126+
}
127+
86128
func Main() int {
87129
flagset := flag.CommandLine
88130
klog.InitFlags(flagset)
@@ -170,10 +212,25 @@ func Main() int {
170212
ctx, cancel := context.WithCancel(context.Background())
171213
defer cancel()
172214

215+
client, err := newClient(ctx, config, *releaseVersion, *namespace, *namespaceUserWorkload)
216+
if err != nil {
217+
fmt.Fprint(os.Stderr, err)
218+
return 1
219+
}
220+
221+
// Retrieve the TLS settings from the API server configuration.
222+
apiServerConfig, err := client.GetAPIServerConfig(ctx)
223+
if err != nil {
224+
fmt.Fprint(os.Stderr, err)
225+
return 1
226+
}
227+
apiServerConfigAdapter := manifests.NewAPIServerConfig(apiServerConfig)
228+
173229
userWorkloadConfigMapName := "user-workload-monitoring-config"
174230
o, err := cmo.New(
175231
ctx,
176-
config,
232+
client,
233+
apiServerConfigAdapter,
177234
*releaseVersion,
178235
*namespace,
179236
*namespaceUserWorkload,
@@ -203,11 +260,23 @@ func Main() int {
203260

204261
wg.Go(func() error { return o.Run(ctx) })
205262

206-
srv, err := server.NewServer("cluster-monitoring-operator", config, *kubeconfigPath, *certFile, *keyFile)
263+
srv, err := server.NewServer(
264+
"cluster-monitoring-operator",
265+
config,
266+
*kubeconfigPath,
267+
*certFile, *keyFile,
268+
apiServerConfigAdapter.MinTLSVersion(),
269+
apiServerConfigAdapter.TLSCiphers(),
270+
)
207271
if err != nil {
208272
fmt.Fprint(os.Stderr, err)
209273
return 1
210274
}
275+
276+
if err := srv.Prepare(ctx); err != nil {
277+
fmt.Fprint(os.Stderr, err)
278+
return 1
279+
}
211280
wg.Go(func() error { return srv.Run(ctx) })
212281

213282
term := make(chan os.Signal, 1)

‎pkg/client/client.go‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -602,8 +602,8 @@ func (c *Client) GetInfrastructure(ctx context.Context, name string) (*configv1.
602602
return c.oscclient.ConfigV1().Infrastructures().Get(ctx, name, metav1.GetOptions{})
603603
}
604604

605-
func (c *Client) GetAPIServerConfig(ctx context.Context, name string) (*configv1.APIServer, error) {
606-
return c.oscclient.ConfigV1().APIServers().Get(ctx, name, metav1.GetOptions{})
605+
func (c *Client) GetAPIServerConfig(ctx context.Context) (*configv1.APIServer, error) {
606+
return c.oscclient.ConfigV1().APIServers().Get(ctx, "cluster", metav1.GetOptions{})
607607
}
608608

609609
func (c *Client) GetConsoleConfig(ctx context.Context, name string) (*configv1.Console, error) {
@@ -1963,3 +1963,11 @@ func Poll(ctx context.Context, condition wait.ConditionWithContextFunc, options
19631963

19641964
return nil
19651965
}
1966+
1967+
func (c *Client) OpenShiftConfigClientset() openshiftconfigclientset.Interface {
1968+
return c.oscclient
1969+
}
1970+
1971+
func (c *Client) EventsRecorder() events.Recorder {
1972+
return c.eventRecorder
1973+
}

‎pkg/operator/operator.go‎

Lines changed: 8 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,10 @@ import (
2626
configv1 "github.com/openshift/api/config/v1"
2727
configv1alpha1 "github.com/openshift/api/config/v1alpha1"
2828
"github.com/openshift/api/features"
29-
configv1client "github.com/openshift/client-go/config/clientset/versioned"
3029
configv1informers "github.com/openshift/client-go/config/informers/externalversions"
3130
"github.com/openshift/library-go/pkg/operator/certrotation"
3231
"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
3332
"github.com/openshift/library-go/pkg/operator/csr"
34-
"github.com/openshift/library-go/pkg/operator/events"
3533
certapiv1 "k8s.io/api/certificates/v1"
3634
v1 "k8s.io/api/core/v1"
3735
apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -40,12 +38,9 @@ import (
4038
apiutilerrors "k8s.io/apimachinery/pkg/util/errors"
4139
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
4240
"k8s.io/client-go/informers"
43-
"k8s.io/client-go/kubernetes"
44-
"k8s.io/client-go/rest"
4541
"k8s.io/client-go/tools/cache"
4642
"k8s.io/client-go/util/workqueue"
4743
"k8s.io/klog/v2"
48-
"k8s.io/utils/clock"
4944
"k8s.io/utils/ptr"
5045

5146
"github.com/openshift/cluster-monitoring-operator/pkg/alert"
@@ -186,9 +181,10 @@ type Operator struct {
186181
remoteWrite bool
187182
ClusterMonitoringConfigEnabled bool
188183

184+
apiServerConfig *manifests.APIServerConfig
185+
189186
lastKnowInfrastructureConfig *InfrastructureConfig
190187
lastKnowProxyConfig *ProxyConfig
191-
lastKnownApiServerConfig *manifests.APIServerConfig
192188
lastKnownConsoleConfig *configv1.Console
193189

194190
client *client.Client
@@ -210,40 +206,14 @@ type Operator struct {
210206

211207
func New(
212208
ctx context.Context,
213-
config *rest.Config,
209+
c *client.Client,
210+
apiServerConfig *manifests.APIServerConfig,
214211
version, namespace, namespaceUserWorkload, configMapName, userWorkloadConfigMapName string,
215212
remoteWrite bool,
216213
images map[string]string,
217214
telemetryMatches []string,
218215
a *manifests.Assets,
219216
) (*Operator, error) {
220-
kclient, err := kubernetes.NewForConfig(config)
221-
if err != nil {
222-
return nil, fmt.Errorf("creating kubernetes clientset client: %w", err)
223-
}
224-
controllerRef, err := events.GetControllerReferenceForCurrentPod(ctx, kclient, namespace, nil)
225-
if err != nil {
226-
klog.Warningf("unable to get owner reference (falling back to namespace): %v", err)
227-
}
228-
229-
eventRecorder := events.NewKubeRecorderWithOptions(
230-
kclient.CoreV1().Events(namespace),
231-
events.RecommendedClusterSingletonCorrelatorOptions(),
232-
"cluster-monitoring-operator",
233-
controllerRef,
234-
clock.RealClock{},
235-
)
236-
237-
configClient, err := configv1client.NewForConfig(config)
238-
if err != nil {
239-
return nil, err
240-
}
241-
242-
c, err := client.NewForConfig(config, version, namespace, namespaceUserWorkload, client.KubernetesClient(kclient), client.OpenshiftConfigClient(configClient), client.EventRecorder(eventRecorder))
243-
if err != nil {
244-
return nil, err
245-
}
246-
247217
ruleController, err := alert.NewRuleController(ctx, c, version)
248218
if err != nil {
249219
return nil, fmt.Errorf("failed to create alerting rule controller: %w", err)
@@ -259,6 +229,7 @@ func New(
259229
telemetryMatches: telemetryMatches,
260230
configMapName: configMapName,
261231
userWorkloadConfigMapName: userWorkloadConfigMapName,
232+
apiServerConfig: apiServerConfig,
262233
remoteWrite: remoteWrite,
263234
namespace: namespace,
264235
namespaceUserWorkload: namespaceUserWorkload,
@@ -432,7 +403,7 @@ func New(
432403
)
433404
o.informerFactories = append(o.informerFactories, kubeInformersOperatorNS)
434405

435-
configInformers := configv1informers.NewSharedInformerFactory(configClient, 10*time.Minute)
406+
configInformers := configv1informers.NewSharedInformerFactory(c.OpenShiftConfigClientset(), 10*time.Minute)
436407
missingVersion := "0.0.1-snapshot"
437408

438409
// By default, when the enabled/disabled list of featuregates changes,
@@ -442,7 +413,7 @@ func New(
442413
version, missingVersion,
443414
configInformers.Config().V1().ClusterVersions(),
444415
configInformers.Config().V1().FeatureGates(),
445-
eventRecorder,
416+
c.EventsRecorder(),
446417
)
447418
go featureGateAccessor.Run(ctx)
448419
go configInformers.Start(ctx.Done())
@@ -802,14 +773,6 @@ func (o *Operator) sync(ctx context.Context, key string) error {
802773

803774
var proxyConfig = getProxyReader(ctx, config, o.loadProxyConfig)
804775

805-
var apiServerConfig *manifests.APIServerConfig
806-
apiServerConfig, err = o.loadApiServerConfig(ctx)
807-
808-
if err != nil {
809-
o.reportFailed(ctx, newRunReportForError("APIServerConfigError", err))
810-
return err
811-
}
812-
813776
consoleConfig, err := o.loadConsoleConfig(ctx)
814777
if err != nil {
815778
klog.Warningf("Fail to load ConsoleConfig, AlertManager's externalURL may be outdated")
@@ -822,7 +785,7 @@ func (o *Operator) sync(ctx context.Context, key string) error {
822785
o.loadInfrastructureConfig(ctx),
823786
proxyConfig,
824787
o.assets,
825-
apiServerConfig,
788+
o.apiServerConfig,
826789
consoleConfig,
827790
)
828791

@@ -959,20 +922,6 @@ func (o *Operator) loadProxyConfig(ctx context.Context) (*ProxyConfig, error) {
959922
return o.lastKnowProxyConfig, nil
960923
}
961924

962-
func (o *Operator) loadApiServerConfig(ctx context.Context) (*manifests.APIServerConfig, error) {
963-
config, err := o.client.GetAPIServerConfig(ctx, "cluster")
964-
if err != nil {
965-
klog.Warningf("failed to get api server config: %v", err)
966-
967-
if o.lastKnownApiServerConfig == nil {
968-
return nil, fmt.Errorf("no last known api server configuration")
969-
}
970-
} else {
971-
o.lastKnownApiServerConfig = manifests.NewAPIServerConfig(config)
972-
}
973-
return o.lastKnownApiServerConfig, nil
974-
}
975-
976925
func (o *Operator) loadConsoleConfig(ctx context.Context) (*configv1.Console, error) {
977926
config, err := o.client.GetConsoleConfig(ctx, "cluster")
978927
if err == nil {

‎pkg/server/server.go‎

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ import (
2424
"github.com/openshift/library-go/pkg/authorization/hardcodedauthorizer"
2525
"github.com/openshift/library-go/pkg/config/configdefaults"
2626
"github.com/openshift/library-go/pkg/config/serving"
27-
"github.com/openshift/library-go/pkg/crypto"
2827
"k8s.io/apiserver/pkg/authorization/authorizer"
2928
"k8s.io/apiserver/pkg/authorization/union"
3029
genericapiserver "k8s.io/apiserver/pkg/server"
@@ -43,10 +42,14 @@ type Server struct {
4342
kubeClient *kubernetes.Clientset
4443
kubeConfig string
4544
certFile, keyFile string
45+
minTLSVersion string
46+
cipherSuites []string
47+
48+
srv *genericapiserver.GenericAPIServer
4649
}
4750

4851
// NewServer returns a functional Server.
49-
func NewServer(name string, config *rest.Config, kubeConfig, certFile, keyFile string) (*Server, error) {
52+
func NewServer(name string, config *rest.Config, kubeConfig, certFile, keyFile string, minTLSVersion string, cipherSuites []string) (*Server, error) {
5053
kubeClient, err := kubernetes.NewForConfig(config)
5154
if err != nil {
5255
return nil, err
@@ -61,24 +64,24 @@ func NewServer(name string, config *rest.Config, kubeConfig, certFile, keyFile s
6164
}, nil
6265
}
6366

64-
// Run starts the HTTPS server exposing the Prometheus /metrics and validate webhook endpoints on port :8443.
67+
// Prepare configures the HTTPS server exposing the Prometheus /metrics and
68+
// validate webhook endpoints on port :8443.
69+
//
6570
// The server performs authn/authz as prescribed by
6671
// https://github.com/openshift/enhancements/blob/master/enhancements/monitoring/client-cert-scraping.md.
67-
func (s *Server) Run(ctx context.Context) error {
72+
func (s *Server) Prepare(ctx context.Context) error {
6873
var server *genericapiserver.GenericAPIServer
6974

7075
servingInfo := configv1.HTTPServingInfo{}
7176
configdefaults.SetRecommendedHTTPServingInfoDefaults(&servingInfo)
7277
servingInfo.ServingInfo.CertInfo.CertFile = s.certFile
7378
servingInfo.ServingInfo.CertInfo.KeyFile = s.keyFile
79+
7480
// Don't set a CA file for client certificates because the CA is read from
7581
// the kube-system/extension-apiserver-authentication ConfigMap.
7682
servingInfo.ServingInfo.ClientCA = ""
77-
// Use intermediate TLS profile cipher suites to avoid insecure cipher warnings
78-
// Convert OpenSSL cipher names to IANA names for Kubernetes validation
79-
intermediateTLSProfile := configv1.TLSProfiles[configv1.TLSProfileIntermediateType]
80-
servingInfo.ServingInfo.CipherSuites = crypto.OpenSSLToIANACipherSuites(intermediateTLSProfile.Ciphers)
81-
servingInfo.ServingInfo.MinTLSVersion = string(intermediateTLSProfile.MinTLSVersion)
83+
servingInfo.ServingInfo.CipherSuites = s.cipherSuites
84+
servingInfo.ServingInfo.MinTLSVersion = s.minTLSVersion
8285

8386
serverConfig, err := serving.ToServerConfig(
8487
ctx,
@@ -125,15 +128,16 @@ func (s *Server) Run(ctx context.Context) error {
125128
*handler,
126129
)
127130

128-
go func() {
129-
if err := server.PrepareRun().RunWithContext(ctx); err != nil {
130-
klog.Fatal(err)
131-
}
132-
klog.Info("server exited")
133-
}()
131+
s.srv = server
132+
return nil
133+
}
134134

135-
<-ctx.Done()
135+
func (s *Server) Run(ctx context.Context) error {
136+
if err := s.srv.PrepareRun().RunWithContext(ctx); ctx.Err() == nil {
137+
return err
138+
}
136139

140+
klog.Info("server exited")
137141
return nil
138142
}
139143

0 commit comments

Comments
 (0)