Skip to content

Commit 1d992e7

Browse files
patrickdillonpatrickdillon
committed
Further refactoring
1 parent 982c39e commit 1d992e7

File tree

6 files changed

+114
-78
lines changed

6 files changed

+114
-78
lines changed

data/data/bootstrap/files/opt/openshift/egress-selector-config.yaml renamed to data/data/bootstrap/files/opt/openshift/egress-selector-config.yaml.template

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
{{- if .KonnectivityEnabled }}
12
apiVersion: apiserver.k8s.io/v1beta1
23
kind: EgressSelectorConfiguration
34
egressSelections:
@@ -13,3 +14,4 @@ egressSelections:
1314
- name: "etcd"
1415
connection:
1516
proxyProtocol: "Direct"
17+
{{- end }}

data/data/bootstrap/files/opt/openshift/konnectivity-config-override.yaml renamed to data/data/bootstrap/files/opt/openshift/konnectivity-config-override.yaml.template

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
1+
{{- if .KonnectivityEnabled }}
12
apiVersion: kubecontrolplane.config.openshift.io/v1
23
kind: KubeAPIServerConfig
34
apiServerArguments:
45
egress-selector-config-file:
56
- "/etc/kubernetes/config/egress-selector-config.yaml"
7+
{{- end }}

‎data/data/bootstrap/files/usr/local/bin/bootkube.sh.template‎

Lines changed: 14 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ set -euoE pipefail ## -E option will cause functions to inherit trap
1010
. /usr/local/bin/bootstrap-cluster-gather.sh
1111
# shellcheck source=bootstrap-verify-api-server-urls.sh
1212
. /usr/local/bin/bootstrap-verify-api-server-urls.sh
13+
# shellcheck source=konnectivity.sh.template
14+
. /usr/local/bin/konnectivity.sh
1315

1416
mkdir --parents /etc/kubernetes/{manifests,bootstrap-configs,bootstrap-manifests}
1517

@@ -19,6 +21,12 @@ BOOTSTRAP_INPLACE=true
1921
BOOTSTRAP_INPLACE=false
2022
{{ end -}}
2123

24+
{{- if .KonnectivityEnabled }}
25+
KONNECTIVITY_ENABLED=true
26+
{{- else }}
27+
KONNECTIVITY_ENABLED=false
28+
{{- end }}
29+
2230
ETCD_ENDPOINTS=
2331

2432
bootkube_podman_run() {
@@ -66,7 +74,6 @@ OPENSHIFT_HYPERKUBE_IMAGE=$(image_for hyperkube)
6674
OPENSHIFT_CLUSTER_POLICY_IMAGE=$(image_for cluster-policy-controller)
6775

6876
CLUSTER_BOOTSTRAP_IMAGE=$(image_for cluster-bootstrap)
69-
KONNECTIVITY_IMAGE=$(image_for apiserver-network-proxy)
7077

7178
mkdir --parents ./{bootstrap-manifests,manifests}
7279

@@ -246,36 +253,7 @@ then
246253
record_service_stage_success
247254
fi
248255

249-
# Detect bootstrap node IP at runtime using the default route source address
250-
# This is needed for Konnectivity agents to connect back to the bootstrap server.
251-
{{- if .UseIPv6ForNodeIP }}
252-
BOOTSTRAP_NODE_IP=$(ip -6 -j route get 2001:4860:4860::8888 | jq -r '.[0].prefsrc')
253-
{{- else }}
254-
BOOTSTRAP_NODE_IP=$(ip -j route get 1.1.1.1 | jq -r '.[0].prefsrc')
255-
{{- end }}
256-
echo "Detected bootstrap node IP: ${BOOTSTRAP_NODE_IP}"
257-
258-
# Generate Konnectivity certificates (self-signed CA, server cert, agent cert)
259-
if [ ! -f konnectivity-certs.done ]
260-
then
261-
record_service_stage_start "konnectivity-certs"
262-
/usr/local/bin/konnectivity-certs.sh "${BOOTSTRAP_NODE_IP}"
263-
touch konnectivity-certs.done
264-
record_service_stage_success
265-
fi
266-
267-
# Create Konnectivity server static pod manifest.
268-
# This runs on the bootstrap node and provides a proxy for the bootstrap KAS
269-
# to reach webhooks in the cluster's pod network.
270-
if [ ! -f konnectivity-server-bootstrap.done ]
271-
then
272-
record_service_stage_start "konnectivity-server-bootstrap"
273-
echo "Creating Konnectivity server static pod manifest..."
274-
export KONNECTIVITY_IMAGE
275-
envsubst < /opt/openshift/konnectivity-server-pod.yaml > /etc/kubernetes/manifests/konnectivity-server-pod.yaml
276-
touch konnectivity-server-bootstrap.done
277-
record_service_stage_success
278-
fi
256+
konnectivity_setup
279257

280258
if [ ! -f kube-apiserver-bootstrap.done ]
281259
then
@@ -305,8 +283,9 @@ then
305283
--config-override-files=/assets/konnectivity-config-override.yaml
306284

307285
cp kube-apiserver-bootstrap/config /etc/kubernetes/bootstrap-configs/kube-apiserver-config.yaml
308-
# Copy egress selector config to bootstrap-configs where KAS can read it
309-
cp /opt/openshift/egress-selector-config.yaml /etc/kubernetes/bootstrap-configs/egress-selector-config.yaml
286+
if [ "${KONNECTIVITY_ENABLED}" = "true" ]; then
287+
cp /opt/openshift/egress-selector-config.yaml /etc/kubernetes/bootstrap-configs/egress-selector-config.yaml
288+
fi
310289
cp kube-apiserver-bootstrap/bootstrap-manifests/* bootstrap-manifests/
311290
cp kube-apiserver-bootstrap/manifests/* manifests/
312291

@@ -601,31 +580,7 @@ then
601580
record_service_stage_success
602581
fi
603582

604-
# Create Konnectivity agent manifests for cluster deployment.
605-
# The namespace, secret, and daemonset templates live as separate files
606-
# under /opt/openshift/ and are populated with runtime values here.
607-
if [ ! -f konnectivity-agent-manifest.done ]
608-
then
609-
record_service_stage_start "konnectivity-agent-manifest"
610-
echo "Creating Konnectivity agent manifests..."
611-
612-
KONNECTIVITY_CERT_DIR=/opt/openshift/tls/konnectivity
613-
614-
# Namespace
615-
cp /opt/openshift/konnectivity-namespace.yaml manifests/konnectivity-namespace.yaml
616-
617-
# Agent certs secret (export base64-encoded cert data for envsubst)
618-
export KONNECTIVITY_AGENT_CERT_BASE64=$(base64 -w0 "${KONNECTIVITY_CERT_DIR}/agent.crt")
619-
export KONNECTIVITY_AGENT_KEY_BASE64=$(base64 -w0 "${KONNECTIVITY_CERT_DIR}/agent.key")
620-
export KONNECTIVITY_CA_CERT_BASE64=$(base64 -w0 "${KONNECTIVITY_CERT_DIR}/ca.crt")
621-
envsubst < /opt/openshift/konnectivity-agent-certs-secret.yaml > manifests/konnectivity-agent-certs.yaml
622-
623-
export BOOTSTRAP_NODE_IP
624-
envsubst < /opt/openshift/konnectivity-agent-daemonset.yaml > manifests/konnectivity-agent-daemonset.yaml
625-
626-
touch konnectivity-agent-manifest.done
627-
record_service_stage_success
628-
fi
583+
konnectivity_manifests
629584

630585
REQUIRED_PODS="openshift-kube-apiserver/kube-apiserver,openshift-kube-scheduler/openshift-kube-scheduler,openshift-kube-controller-manager/kube-controller-manager,openshift-cluster-version/cluster-version-operator"
631586
if [ "$BOOTSTRAP_INPLACE" = true ]
@@ -712,18 +667,7 @@ if [ ! -f api-int-dns-check.done ]; then
712667
fi
713668
fi
714669

715-
# Clean up bootstrap konnectivity resources by deleting the namespace.
716-
# This cascades to all resources (DaemonSet, Secret) within it.
717-
if [ ! -f konnectivity-cleanup.done ]; then
718-
record_service_stage_start "konnectivity-cleanup"
719-
echo "Cleaning up bootstrap konnectivity resources..."
720-
oc delete namespace openshift-bootstrap-konnectivity \
721-
--kubeconfig=/opt/openshift/auth/kubeconfig \
722-
--ignore-not-found=true || true
723-
rm -f /etc/kubernetes/manifests/konnectivity-server-pod.yaml
724-
touch konnectivity-cleanup.done
725-
record_service_stage_success
726-
fi
670+
konnectivity_cleanup
727671

728672
# Workaround for https://github.com/opencontainers/runc/pull/1807
729673
touch /opt/openshift/.bootkube.done

‎data/data/bootstrap/files/usr/local/bin/konnectivity.sh.template‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
#!/usr/bin/env bash
2+
# Konnectivity bootstrap functions.
3+
# Sourced by bootkube.sh — do not execute directly.
4+
5+
# konnectivity_setup detects the bootstrap node IP, generates certificates,
6+
# and creates the konnectivity server static pod manifest.
7+
konnectivity_setup() {
8+
if [ "${KONNECTIVITY_ENABLED}" != "true" ]; then
9+
return 0
10+
fi
11+
12+
# Detect bootstrap node IP at runtime using the default route source address.
13+
# Konnectivity agents use this to connect back to the bootstrap server.
14+
{{- if .UseIPv6ForNodeIP }}
15+
BOOTSTRAP_NODE_IP=$(ip -6 -j route get 2001:4860:4860::8888 | jq -r '.[0].prefsrc')
16+
{{- else }}
17+
BOOTSTRAP_NODE_IP=$(ip -j route get 1.1.1.1 | jq -r '.[0].prefsrc')
18+
{{- end }}
19+
echo "Detected bootstrap node IP: ${BOOTSTRAP_NODE_IP}"
20+
21+
if [ ! -f konnectivity-certs.done ]; then
22+
record_service_stage_start "konnectivity-certs"
23+
/usr/local/bin/konnectivity-certs.sh "${BOOTSTRAP_NODE_IP}"
24+
touch konnectivity-certs.done
25+
record_service_stage_success
26+
fi
27+
28+
if [ ! -f konnectivity-server-bootstrap.done ]; then
29+
record_service_stage_start "konnectivity-server-bootstrap"
30+
echo "Creating Konnectivity server static pod manifest..."
31+
export KONNECTIVITY_IMAGE=$(image_for apiserver-network-proxy)
32+
envsubst < /opt/openshift/konnectivity-server-pod.yaml > /etc/kubernetes/manifests/konnectivity-server-pod.yaml
33+
touch konnectivity-server-bootstrap.done
34+
record_service_stage_success
35+
fi
36+
}
37+
38+
# konnectivity_manifests creates the agent namespace, secret, and daemonset
39+
# manifests for cluster deployment.
40+
konnectivity_manifests() {
41+
if [ "${KONNECTIVITY_ENABLED}" != "true" ]; then
42+
return 0
43+
fi
44+
45+
if [ ! -f konnectivity-agent-manifest.done ]; then
46+
record_service_stage_start "konnectivity-agent-manifest"
47+
echo "Creating Konnectivity agent manifests..."
48+
49+
KONNECTIVITY_CERT_DIR=/opt/openshift/tls/konnectivity
50+
51+
cp /opt/openshift/konnectivity-namespace.yaml manifests/konnectivity-namespace.yaml
52+
53+
export KONNECTIVITY_AGENT_CERT_BASE64=$(base64 -w0 "${KONNECTIVITY_CERT_DIR}/agent.crt")
54+
export KONNECTIVITY_AGENT_KEY_BASE64=$(base64 -w0 "${KONNECTIVITY_CERT_DIR}/agent.key")
55+
export KONNECTIVITY_CA_CERT_BASE64=$(base64 -w0 "${KONNECTIVITY_CERT_DIR}/ca.crt")
56+
envsubst < /opt/openshift/konnectivity-agent-certs-secret.yaml > manifests/konnectivity-agent-certs.yaml
57+
58+
export BOOTSTRAP_NODE_IP
59+
envsubst < /opt/openshift/konnectivity-agent-daemonset.yaml > manifests/konnectivity-agent-daemonset.yaml
60+
61+
touch konnectivity-agent-manifest.done
62+
record_service_stage_success
63+
fi
64+
}
65+
66+
# konnectivity_cleanup removes bootstrap konnectivity resources by deleting
67+
# the namespace (cascading to DaemonSet and Secret) and the server static pod.
68+
konnectivity_cleanup() {
69+
if [ "${KONNECTIVITY_ENABLED}" != "true" ]; then
70+
return 0
71+
fi
72+
73+
if [ ! -f konnectivity-cleanup.done ]; then
74+
record_service_stage_start "konnectivity-cleanup"
75+
echo "Cleaning up bootstrap konnectivity resources..."
76+
oc delete namespace openshift-bootstrap-konnectivity \
77+
--kubeconfig=/opt/openshift/auth/kubeconfig \
78+
--ignore-not-found=true || true
79+
rm -f /etc/kubernetes/manifests/konnectivity-server-pod.yaml
80+
touch konnectivity-cleanup.done
81+
record_service_stage_success
82+
fi
83+
}

‎pkg/asset/ignition/bootstrap/common.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"k8s.io/utils/ptr"
2626

2727
configv1 "github.com/openshift/api/config/v1"
28+
"github.com/openshift/api/features"
2829
"github.com/openshift/installer/data"
2930
"github.com/openshift/installer/pkg/asset"
3031
"github.com/openshift/installer/pkg/asset/ignition"
@@ -97,6 +98,7 @@ type bootstrapTemplateData struct {
9798
FeatureSet configv1.FeatureSet
9899
Invoker string
99100
ClusterDomain string
101+
KonnectivityEnabled bool
100102
}
101103

102104
// platformTemplateData is the data to use to replace values in bootstrap
@@ -401,6 +403,7 @@ func (a *Common) getTemplateData(dependencies asset.Parents, bootstrapInPlace bo
401403
FeatureSet: installConfig.Config.FeatureSet,
402404
Invoker: openshiftInstallInvoker,
403405
ClusterDomain: installConfig.Config.ClusterDomain(),
406+
KonnectivityEnabled: !bootstrapInPlace && installConfig.Config.EnabledFeatureGates().Enabled(features.FeatureGateMachineAPIMigrationAWS),
404407
}
405408
}
406409

‎pkg/gather/service/analyze.go‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -114,18 +114,20 @@ func checkReleaseImageDownload(a analysis) bool {
114114
return false
115115
}
116116

117-
// bootstrap-verify-api-servel-urls.sh is currently running as part of the bootkube service.
118-
// And the verification of the API and API-Int URLs are the only stage where a failure is
119-
// currently reported. So, here we are able to conclude that a failure corresponds to a
120-
// failure to resolve either the API URL or API-Int URL or both. If that changes and if
121-
// any other stage in the bootkube service starts reporting a failure, we need to revisit
122-
// this. At that point verification of the URLs could be moved to its own service.
123117
func checkBootkubeService(a analysis) bool {
124118
if a.successful {
125119
return true
126120
}
127-
// Note: Even when there is a stage failure, we are not returning false here. That is
128-
// intentional because we donot want to report this as an error in the "analyze" output.
121+
switch a.failingStage {
122+
case "konnectivity-certs":
123+
logrus.Error("The bootstrap machine failed to generate konnectivity certificates")
124+
case "konnectivity-server-bootstrap":
125+
logrus.Error("The bootstrap machine failed to start the konnectivity server")
126+
case "konnectivity-agent-manifest":
127+
logrus.Error("The bootstrap machine failed to create konnectivity agent manifests")
128+
case "konnectivity-cleanup":
129+
logrus.Error("The bootstrap machine failed to clean up konnectivity resources")
130+
}
129131
a.logLastError()
130132
return true
131133
}

0 commit comments

Comments
 (0)