Skip to content

Commit d510ad1

Browse files
tmshorttmshort
authored
✨ Certificate support for image registry (#960)
* Certificate support for image registry Remove the InsecureSkipTLSVerify annotations. * Create a ClusterIssuer CA (via openssl) that is used by OLMv1 e2e * Update the operator controller to specify a cert directory, rather than a single file. * Use this directory for catalogd and image-registries * Update the deployment to reference CAs appropriately Signed-off-by: Todd Short <[email protected]> * fixup! Certificate support for image registry Signed-off-by: Todd Short <[email protected]> * fixup! Certificate support for image registry Signed-off-by: Todd Short <[email protected]> * fixup! Certificate support for image registry Signed-off-by: Todd Short <[email protected]> * fixup! Certificate support for image registry * fixup! Certificate support for image registry * fixup! Certificate support for image registry --------- Signed-off-by: Todd Short <[email protected]>
1 parent 6c61a78 commit d510ad1

File tree

13 files changed

+176
-60
lines changed

13 files changed

+176
-60
lines changed

‎Tiltfile

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
if not os.path.exists('../tilt-support'):
22
fail('Please clone https://github.com/operator-framework/tilt-support to ../tilt-support')
33

4-
load('../tilt-support/Tiltfile', 'deploy_repo')
4+
load('../tilt-support/Tiltfile', 'deploy_repo', 'process_yaml')
55

66
config.define_string_list('repos', args=True)
77
cfg = config.parse()
@@ -16,6 +16,8 @@ repo = {
1616
'starting_debug_port': 30000,
1717
}
1818

19+
process_yaml("testdata/certs/issuers.yaml")
20+
1921
for r in repos:
2022
if r == 'operator-controller':
2123
deploy_repo('operator-controller', repo)

‎cmd/manager/main.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -80,11 +80,11 @@ func main() {
8080
cachePath string
8181
operatorControllerVersion bool
8282
systemNamespace string
83-
caCert string
83+
caCertDir string
8484
)
8585
flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
8686
flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
87-
flag.StringVar(&caCert, "ca-cert", "", "The TLS certificate to use for verifying HTTPS connections to the Catalogd web server.")
87+
flag.StringVar(&caCertDir, "ca-certs-dir", "", "The directory of TLS certificate to use for verifying HTTPS connections to the Catalogd and docker-registry web servers.")
8888
flag.BoolVar(&enableLeaderElection, "leader-elect", false,
8989
"Enable leader election for controller manager. "+
9090
"Enabling this will ensure there is only one active controller manager.")
@@ -153,7 +153,7 @@ func main() {
153153
os.Exit(1)
154154
}
155155

156-
httpClient, err := httputil.BuildHTTPClient(caCert)
156+
httpClient, err := httputil.BuildHTTPClient(caCertDir)
157157
if err != nil {
158158
setupLog.Error(err, "unable to create catalogd http client")
159159
}
@@ -224,6 +224,7 @@ func main() {
224224
InstalledBundleGetter: &controllers.DefaultInstalledBundleGetter{ActionClientGetter: acg},
225225
Handler: registryv1handler.HandlerFunc(registry.HandleBundleDeployment),
226226
Finalizers: clusterExtensionFinalizers,
227+
CaCertDir: caCertDir,
227228
}).SetupWithManager(mgr); err != nil {
228229
setupLog.Error(err, "unable to create controller", "controller", "ClusterExtension")
229230
os.Exit(1)

‎config/overlays/tls/kustomization.yaml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1
1212
kind: Kustomization
1313
resources:
1414
- ../../base
15+
- resources/manager_cert.yaml
1516

1617
patches:
1718
- target:
1819
kind: Deployment
1920
name: controller-manager
20-
path: patches/manager_deployment_cert.yaml
21+
path: patches/manager_deployment_cert.yaml
22+
- path: patches/manager_cert_patch.yaml

‎config/overlays/tls/patches/manager_cert_patch.yaml

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: controller-manager
5+
namespace: system
6+
spec:
7+
template:
8+
spec:
9+
containers:
10+
- name: kube-rbac-proxy
11+
- name: manager
12+
volumeMounts:
13+
- name: e2e-cert
14+
mountPath: /var/certs/olm-ca.crt
15+
subPath: olm-ca.crt
16+
readOnly: true
17+
volumes:
18+
- name: e2e-cert
19+
secret:
20+
secretName: olmv1-cert
21+
items:
22+
- key: ca.crt
23+
path: olm-ca.crt

‎config/overlays/tls/patches/manager_deployment_cert.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
- op: add
22
path: /spec/template/spec/volumes/-
3-
value: {"name":"ca-certificate", "secret":{"secretName":"catalogd-catalogserver-cert", "optional": false, "items": [{"key": "tls.crt", "path": "tls.crt"}]}}
3+
value: {"name":"catalogd-certificate", "secret":{"secretName":"catalogd-catalogserver-cert", "optional": false, "items": [{"key": "ca.crt", "path": "catalogd.crt"}]}}
44
- op: add
55
path: /spec/template/spec/containers/0/volumeMounts/-
6-
value: {"name":"ca-certificate", "readOnly": true, "mountPath":"/var/certs"}
6+
value: {"name":"catalogd-certificate", "readOnly": true, "mountPath":"/var/certs/catalogd.crt", "subPath":"catalogd.crt"}
77
- op: add
88
path: /spec/template/spec/containers/0/args/-
9-
value: "--ca-cert=/var/certs/tls.crt"
9+
value: "--ca-certs-dir=/var/certs"

‎config/overlays/tls/resources/manager_cert.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
apiVersion: cert-manager.io/v1
2+
kind: Certificate
3+
metadata:
4+
name: olmv1-cert
5+
spec:
6+
secretName: olmv1-cert
7+
dnsNames:
8+
- operator-controller.olmv1-system.svc
9+
- operator-controller.olmv1-system.svc.cluster.local
10+
privateKey:
11+
algorithm: ECDSA
12+
size: 256
13+
issuerRef:
14+
name: olmv1-ca
15+
kind: ClusterIssuer
16+
group: cert-manager.io

‎internal/controllers/clusterextension_controller.go

Lines changed: 10 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ import (
7373
catalogfilter "github.com/operator-framework/operator-controller/internal/catalogmetadata/filter"
7474
catalogsort "github.com/operator-framework/operator-controller/internal/catalogmetadata/sort"
7575
"github.com/operator-framework/operator-controller/internal/conditionsets"
76+
"github.com/operator-framework/operator-controller/internal/httputil"
7677
"github.com/operator-framework/operator-controller/internal/labels"
7778
)
7879

@@ -90,16 +91,13 @@ type ClusterExtensionReconciler struct {
9091
cache cache.Cache
9192
InstalledBundleGetter InstalledBundleGetter
9293
Finalizers crfinalizer.Finalizers
94+
CaCertDir string
9395
}
9496

9597
type InstalledBundleGetter interface {
9698
GetInstalledBundle(ctx context.Context, ext *ocv1alpha1.ClusterExtension) (*ocv1alpha1.BundleMetadata, error)
9799
}
98100

99-
const (
100-
bundleConnectionAnnotation string = "bundle.connection.config/insecureSkipTLSVerify"
101-
)
102-
103101
//+kubebuilder:rbac:groups=olm.operatorframework.io,resources=clusterextensions,verbs=get;list;watch
104102
//+kubebuilder:rbac:groups=olm.operatorframework.io,resources=clusterextensions/status,verbs=update;patch
105103
//+kubebuilder:rbac:groups=olm.operatorframework.io,resources=clusterextensions/finalizers,verbs=update
@@ -249,7 +247,7 @@ func (r *ClusterExtensionReconciler) reconcile(ctx context.Context, ext *ocv1alp
249247
// Generate a BundleDeployment from the ClusterExtension to Unpack.
250248
// Note: The BundleDeployment here is not a k8s API, its a simple Go struct which
251249
// necessary embedded values.
252-
bd := r.generateBundleDeploymentForUnpack(bundle.Image, ext)
250+
bd := r.generateBundleDeploymentForUnpack(ctx, bundle.Image, ext)
253251
unpackResult, err := r.Unpacker.Unpack(ctx, bd)
254252
if err != nil {
255253
setStatusUnpackFailed(ext, err.Error())
@@ -532,7 +530,11 @@ func SetDeprecationStatus(ext *ocv1alpha1.ClusterExtension, bundle *catalogmetad
532530
}
533531
}
534532

535-
func (r *ClusterExtensionReconciler) generateBundleDeploymentForUnpack(bundlePath string, ce *ocv1alpha1.ClusterExtension) *rukpakv1alpha2.BundleDeployment {
533+
func (r *ClusterExtensionReconciler) generateBundleDeploymentForUnpack(ctx context.Context, bundlePath string, ce *ocv1alpha1.ClusterExtension) *rukpakv1alpha2.BundleDeployment {
534+
certData, err := httputil.LoadCerts(r.CaCertDir)
535+
if err != nil {
536+
log.FromContext(ctx).WithName("operator-controller").WithValues("cluster-extension", ce.GetName()).Error(err, "unable to get TLS certificate")
537+
}
536538
return &rukpakv1alpha2.BundleDeployment{
537539
TypeMeta: metav1.TypeMeta{
538540
Kind: ce.Kind,
@@ -547,25 +549,14 @@ func (r *ClusterExtensionReconciler) generateBundleDeploymentForUnpack(bundlePat
547549
Source: rukpakv1alpha2.BundleSource{
548550
Type: rukpakv1alpha2.SourceTypeImage,
549551
Image: &rukpakv1alpha2.ImageSource{
550-
Ref: bundlePath,
551-
InsecureSkipTLSVerify: isInsecureSkipTLSVerifySet(ce),
552+
Ref: bundlePath,
553+
CertificateData: certData,
552554
},
553555
},
554556
},
555557
}
556558
}
557559

558-
func isInsecureSkipTLSVerifySet(ce *ocv1alpha1.ClusterExtension) bool {
559-
if ce == nil {
560-
return false
561-
}
562-
value, ok := ce.Annotations[bundleConnectionAnnotation]
563-
if !ok {
564-
return false
565-
}
566-
return value == "true"
567-
}
568-
569560
// SetupWithManager sets up the controller with the Manager.
570561
func (r *ClusterExtensionReconciler) SetupWithManager(mgr ctrl.Manager) error {
571562
controller, err := ctrl.NewControllerManagedBy(mgr).

‎internal/httputil/httputil.go

Lines changed: 44 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -5,30 +5,57 @@ import (
55
"crypto/x509"
66
"net/http"
77
"os"
8+
"path/filepath"
9+
"strings"
810
"time"
911
)
1012

11-
func BuildHTTPClient(caCert string) (*http.Client, error) {
12-
httpClient := &http.Client{Timeout: 10 * time.Second}
13-
14-
if caCert != "" {
15-
// tlsFileWatcher, err := certwatcher.New(caCert, "")
13+
func LoadCerts(caDir string) (string, error) {
14+
if caDir == "" {
15+
return "", nil
16+
}
1617

17-
cert, err := os.ReadFile(caCert)
18-
if err != nil {
19-
return nil, err
20-
}
21-
caCertPool := x509.NewCertPool()
22-
caCertPool.AppendCertsFromPEM(cert)
23-
tlsConfig := &tls.Config{
24-
RootCAs: caCertPool,
25-
MinVersion: tls.VersionTLS12,
18+
certs := []string{}
19+
dirEntries, err := os.ReadDir(caDir)
20+
if err != nil {
21+
return "", err
22+
}
23+
for _, e := range dirEntries {
24+
if e.IsDir() {
25+
continue
2626
}
27-
tlsTransport := &http.Transport{
28-
TLSClientConfig: tlsConfig,
27+
data, err := os.ReadFile(filepath.Join(caDir, e.Name()))
28+
if err != nil {
29+
return "", err
2930
}
30-
httpClient.Transport = tlsTransport
31+
certs = append(certs, string(data))
32+
}
33+
return strings.Join(certs, "\n"), nil
34+
}
35+
36+
func BuildHTTPClient(caDir string) (*http.Client, error) {
37+
httpClient := &http.Client{Timeout: 10 * time.Second}
38+
39+
// use the SystemCertPool as a default
40+
caCertPool, err := x509.SystemCertPool()
41+
if err != nil {
42+
return nil, err
43+
}
44+
45+
certs, err := LoadCerts(caDir)
46+
if err != nil {
47+
return nil, err
48+
}
49+
50+
caCertPool.AppendCertsFromPEM([]byte(certs))
51+
tlsConfig := &tls.Config{
52+
RootCAs: caCertPool,
53+
MinVersion: tls.VersionTLS12,
54+
}
55+
tlsTransport := &http.Transport{
56+
TLSClientConfig: tlsConfig,
3157
}
58+
httpClient.Transport = tlsTransport
3259

3360
return httpClient, nil
3461
}

‎scripts/install.tpl.sh

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,42 @@ function kubectl_wait() {
3535
kubectl apply -f "https://github.com/cert-manager/cert-manager/releases/download/${cert_mgr_version}/cert-manager.yaml"
3636
kubectl_wait "cert-manager" "deployment/cert-manager-webhook" "60s"
3737

38+
# Create a self-signed ClusterIssuer
39+
kubectl apply -f - <<EOF
40+
apiVersion: cert-manager.io/v1
41+
kind: Issuer
42+
metadata:
43+
name: self-sign-issuer
44+
namespace: cert-manager
45+
spec:
46+
selfSigned: {}
47+
---
48+
apiVersion: cert-manager.io/v1
49+
kind: Certificate
50+
metadata:
51+
name: olmv1-ca
52+
namespace: cert-manager
53+
spec:
54+
isCA: true
55+
commonName: olmv1-ca
56+
secretName: olmv1-ca
57+
privateKey:
58+
algorithm: ECDSA
59+
size: 256
60+
issuerRef:
61+
name: self-sign-issuer
62+
kind: Issuer
63+
group: cert-manager.io
64+
---
65+
apiVersion: cert-manager.io/v1
66+
kind: ClusterIssuer
67+
metadata:
68+
name: olmv1-ca
69+
spec:
70+
ca:
71+
secretName: olmv1-ca
72+
EOF
73+
3874
kubectl apply -f "https://github.com/operator-framework/catalogd/releases/download/${catalogd_version}/catalogd.yaml"
3975
kubectl_wait "olmv1-system" "deployment/catalogd-controller-manager" "60s"
4076

‎test/e2e/cluster_extension_install_test.go

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,6 @@ func testInit(t *testing.T) (*ocv1alpha1.ClusterExtension, *catalogd.ClusterCata
4545
clusterExtension := &ocv1alpha1.ClusterExtension{
4646
ObjectMeta: metav1.ObjectMeta{
4747
Name: clusterExtensionName,
48-
Annotations: map[string]string{
49-
"bundle.connection.config/insecureSkipTLSVerify": "true",
50-
},
5148
},
5249
}
5350
return clusterExtension, extensionCatalog

0 commit comments

Comments
 (0)