Skip to content

Commit ac24089

Browse files
tiansuo114tiansuo114
committed
feat: karmadactl support split secret layout in init command
Signed-off-by: tiansuo <[email protected]> fix Signed-off-by: tiansuo <[email protected]> fix Signed-off-by: tiansuo <[email protected]> fix Signed-off-by: tiansuo <[email protected]>
1 parent 04adc23 commit ac24089

File tree

11 files changed

+1180
-188
lines changed

11 files changed

+1180
-188
lines changed

‎.github/workflows/installation-cli.yaml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,3 +106,47 @@ jobs:
106106
with:
107107
name: karmadactl_config_test_logs_${{ matrix.k8s }}
108108
path: ${{ github.workspace }}/karmadactl-test-logs/${{ matrix.k8s }}/config/
109+
110+
test-on-kubernetes-split-secret:
111+
name: Test on Kubernetes (Split Secret)
112+
runs-on: ubuntu-22.04
113+
strategy:
114+
fail-fast: false
115+
matrix:
116+
# Latest three minor releases of Kubernetes
117+
k8s: [ v1.31.0, v1.32.0, v1.33.0 ]
118+
steps:
119+
- name: checkout code
120+
uses: actions/checkout@v5
121+
with:
122+
fetch-depth: 0
123+
- name: install Go
124+
uses: actions/setup-go@v6
125+
with:
126+
go-version-file: go.mod
127+
- name: run karmadactl init test (split secret layout)
128+
run: |
129+
export CLUSTER_VERSION=kindest/node:${{ matrix.k8s }}
130+
131+
# init e2e environment with split secret layout
132+
hack/cli-testing-environment-split-secret.sh
133+
134+
# run a single e2e
135+
export PULL_BASED_CLUSTERS="split-secret-member1:${HOME}/.kube/split-secret-member1.config"
136+
export KUBECONFIG=${HOME}/.kube/karmada-host.config:${HOME}/karmada/karmada-apiserver.config
137+
GO111MODULE=on go install github.com/onsi/ginkgo/v2/ginkgo
138+
ginkgo -v --race --trace -p --focus="[BasicPropagation] propagation testing deployment propagation testing" ./test/e2e/suites/base
139+
- name: export logs (split secret)
140+
if: always()
141+
run: |
142+
export ARTIFACTS_PATH=${{ github.workspace }}/karmadactl-test-logs/${{ matrix.k8s }}/split-secret
143+
mkdir -p $ARTIFACTS_PATH
144+
145+
mkdir -p $ARTIFACTS_PATH/karmada-host
146+
kind export logs --name=karmada-host $ARTIFACTS_PATH/karmada-host
147+
- name: upload logs (split secret)
148+
if: always()
149+
uses: actions/upload-artifact@v4
150+
with:
151+
name: karmadactl_split_secret_test_logs_${{ matrix.k8s }}
152+
path: ${{ github.workspace }}/karmadactl-test-logs/${{ matrix.k8s }}/split-secret/

‎hack/cli-testing-environment-split-secret.sh‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2025 The Karmada Authors.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
set -o errexit
17+
set -o nounset
18+
set -o pipefail
19+
20+
# This script starts a local karmada control plane with karmadactl and with a certain number of clusters joined,
21+
# identical to hack/cli-testing-environment.sh except adding '--secret-layout=split' for init.
22+
# This script depends on utils in: ${REPO_ROOT}/hack/util.sh
23+
# 1. used by developer to setup develop environment quickly.
24+
# 2. used by e2e testing to setup test environment automatically.
25+
26+
REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
27+
source "${REPO_ROOT}"/hack/util.sh
28+
29+
# variable define
30+
KUBECONFIG_PATH=${KUBECONFIG_PATH:-"${HOME}/.kube"}
31+
HOST_CLUSTER_NAME=${HOST_CLUSTER_NAME:-"karmada-host"}
32+
MEMBER_CLUSTER_1_NAME=${MEMBER_CLUSTER_1_NAME:-"split-secret-member1"}
33+
MEMBER_CLUSTER_2_NAME=${MEMBER_CLUSTER_2_NAME:-"split-secret-member2"}
34+
CLUSTER_VERSION=${CLUSTER_VERSION:-"${DEFAULT_CLUSTER_VERSION}"}
35+
BUILD_PATH=${BUILD_PATH:-"_output/bin/linux/amd64"}
36+
37+
# install kind and kubectl
38+
echo -n "Preparing: 'kind' existence check - "
39+
if util::cmd_exist kind; then
40+
echo "passed"
41+
else
42+
echo "not pass"
43+
# Install kind using the version defined in util.sh
44+
util::install_tools "sigs.k8s.io/kind" "${KIND_VERSION}"
45+
fi
46+
# get arch name and os name in bootstrap
47+
BS_ARCH=$(go env GOARCH)
48+
BS_OS=$(go env GOOS)
49+
# check arch and os name before installing
50+
util::install_environment_check "${BS_ARCH}" "${BS_OS}"
51+
echo -n "Preparing: 'kubectl' existence check - "
52+
if util::cmd_exist kubectl; then
53+
echo "passed"
54+
else
55+
echo "not pass"
56+
util::install_kubectl "" "${BS_ARCH}" "${BS_OS}"
57+
fi
58+
59+
# prepare the newest crds
60+
echo "Prepare the newest crds"
61+
cd charts/karmada/
62+
cp -r _crds crds
63+
tar -zcvf ../../crds.tar.gz crds
64+
cd -
65+
66+
# make images
67+
export VERSION="latest"
68+
export REGISTRY="docker.io/karmada"
69+
make images GOOS="linux" --directory="${REPO_ROOT}"
70+
71+
# make karmadactl binary
72+
make karmadactl
73+
74+
# create host/member1/member2 cluster
75+
echo "Start create clusters..."
76+
hack/create-cluster.sh ${HOST_CLUSTER_NAME} ${KUBECONFIG_PATH}/${HOST_CLUSTER_NAME}.config > /dev/null 2>&1 &
77+
hack/create-cluster.sh ${MEMBER_CLUSTER_1_NAME} ${KUBECONFIG_PATH}/${MEMBER_CLUSTER_1_NAME}.config > /dev/null 2>&1 &
78+
hack/create-cluster.sh ${MEMBER_CLUSTER_2_NAME} ${KUBECONFIG_PATH}/${MEMBER_CLUSTER_2_NAME}.config > /dev/null 2>&1 &
79+
80+
# wait cluster ready
81+
echo "Wait clusters ready..."
82+
util::wait_file_exist ${KUBECONFIG_PATH}/${HOST_CLUSTER_NAME}.config 300
83+
util::wait_context_exist ${HOST_CLUSTER_NAME} ${KUBECONFIG_PATH}/${HOST_CLUSTER_NAME}.config 300
84+
kubectl wait --for=condition=Ready nodes --all --timeout=800s --kubeconfig=${KUBECONFIG_PATH}/${HOST_CLUSTER_NAME}.config
85+
util::wait_nodes_taint_disappear 800 ${KUBECONFIG_PATH}/${HOST_CLUSTER_NAME}.config
86+
87+
util::wait_file_exist ${KUBECONFIG_PATH}/${MEMBER_CLUSTER_1_NAME}.config 300
88+
util::wait_context_exist "${MEMBER_CLUSTER_1_NAME}" ${KUBECONFIG_PATH}/${MEMBER_CLUSTER_1_NAME}.config 300
89+
kubectl wait --for=condition=Ready nodes --all --timeout=800s --kubeconfig=${KUBECONFIG_PATH}/${MEMBER_CLUSTER_1_NAME}.config
90+
util::wait_nodes_taint_disappear 800 ${KUBECONFIG_PATH}/${MEMBER_CLUSTER_1_NAME}.config
91+
92+
util::wait_file_exist ${KUBECONFIG_PATH}/${MEMBER_CLUSTER_2_NAME}.config 300
93+
util::wait_context_exist "${MEMBER_CLUSTER_2_NAME}" ${KUBECONFIG_PATH}/${MEMBER_CLUSTER_2_NAME}.config 300
94+
kubectl wait --for=condition=Ready nodes --all --timeout=800s --kubeconfig=${KUBECONFIG_PATH}/${MEMBER_CLUSTER_2_NAME}.config
95+
util::wait_nodes_taint_disappear 800 ${KUBECONFIG_PATH}/${MEMBER_CLUSTER_2_NAME}.config
96+
97+
# load components images to kind cluster
98+
kind load docker-image "${REGISTRY}/karmada-controller-manager:${VERSION}" --name="${HOST_CLUSTER_NAME}"
99+
kind load docker-image "${REGISTRY}/karmada-scheduler:${VERSION}" --name="${HOST_CLUSTER_NAME}"
100+
kind load docker-image "${REGISTRY}/karmada-webhook:${VERSION}" --name="${HOST_CLUSTER_NAME}"
101+
kind load docker-image "${REGISTRY}/karmada-aggregated-apiserver:${VERSION}" --name="${HOST_CLUSTER_NAME}"
102+
kind load docker-image "${REGISTRY}/karmada-agent:${VERSION}" --name="${MEMBER_CLUSTER_1_NAME}"
103+
104+
# init Karmada control plane
105+
echo "Start init karmada control plane..."
106+
${BUILD_PATH}/karmadactl init --kubeconfig=${KUBECONFIG_PATH}/${HOST_CLUSTER_NAME}.config \
107+
--karmada-controller-manager-image="${REGISTRY}/karmada-controller-manager:${VERSION}" \
108+
--karmada-scheduler-image="${REGISTRY}/karmada-scheduler:${VERSION}" \
109+
--karmada-webhook-image="${REGISTRY}/karmada-webhook:${VERSION}" \
110+
--karmada-aggregated-apiserver-image="${REGISTRY}/karmada-aggregated-apiserver:${VERSION}" \
111+
--karmada-data=${HOME}/karmada \
112+
--karmada-pki=${HOME}/karmada/pki \
113+
--crds=./crds.tar.gz \
114+
--secret-layout='split'
115+
116+
# join cluster
117+
echo "Join member clusters..."
118+
TOKEN_CMD=$(${BUILD_PATH}/karmadactl --kubeconfig ${HOME}/karmada/karmada-apiserver.config token create --print-register-command)
119+
TOKEN=$(echo "$TOKEN_CMD" | grep -o '\--token [^ ]*' | cut -d' ' -f2)
120+
HASH=$(echo "$TOKEN_CMD" | grep -o '\--discovery-token-ca-cert-hash [^ ]*' | cut -d' ' -f2)
121+
ENDPOINT=$(kubectl --kubeconfig ${HOME}/karmada/karmada-apiserver.config config view --minify -o jsonpath='{.clusters[0].cluster.server}' | sed 's|^https://||')
122+
123+
${BUILD_PATH}/karmadactl register ${ENDPOINT} \
124+
--token ${TOKEN} \
125+
--discovery-token-ca-cert-hash ${HASH} \
126+
--kubeconfig=${KUBECONFIG_PATH}/${MEMBER_CLUSTER_1_NAME}.config \
127+
--cluster-name=${MEMBER_CLUSTER_1_NAME} \
128+
--karmada-agent-image "${REGISTRY}/karmada-agent:${VERSION}" \
129+
--v=4
130+
131+
${BUILD_PATH}/karmadactl --kubeconfig ${HOME}/karmada/karmada-apiserver.config join ${MEMBER_CLUSTER_2_NAME} --cluster-kubeconfig=${KUBECONFIG_PATH}/${MEMBER_CLUSTER_2_NAME}.config
132+
133+
kubectl wait --for=condition=Ready clusters --all --timeout=800s --kubeconfig=${HOME}/karmada/karmada-apiserver.config

‎pkg/cert/constants.go‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
/*
2+
Copyright 2025 The Karmada Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package cert
18+
19+
// Package cert centralizes TLS-related constants (secret names and key names)
20+
// so they can be shared by cmdinit and future operator implementations.
21+
22+
// Secret names for split-layout TLS materials
23+
// nolint:gosec // These are Kubernetes Secret resource names, not credentials.
24+
const (
25+
// apiserver
26+
SecretApiserverServer = "karmada-apiserver-cert"
27+
SecretApiserverEtcdClient = "karmada-apiserver-etcd-client-cert"
28+
SecretApiserverFrontProxyClient = "karmada-apiserver-front-proxy-client-cert"
29+
SecretApiserverServiceAccountKeys = "karmada-apiserver-service-account-key-pair"
30+
31+
// aggregated apiserver
32+
SecretAggregatedAPIServerServer = "karmada-aggregated-apiserver-cert"
33+
SecretAggregatedAPIServerEtcdClient = "karmada-aggregated-apiserver-etcd-client-cert"
34+
35+
// kube-controller-manager
36+
SecretKubeControllerManagerCA = "kube-controller-manager-ca-cert"
37+
SecretKubeControllerManagerSAKeys = "kube-controller-manager-service-account-key-pair"
38+
39+
// scheduler(estimator) clients
40+
SecretSchedulerEstimatorClient = "karmada-scheduler-scheduler-estimator-client-cert"
41+
SecretDeschedulerEstimatorClient = "karmada-descheduler-scheduler-estimator-client-cert"
42+
43+
// etcd (internal)
44+
SecretEtcdServer = "etcd-cert"
45+
SecretEtcdClient = "etcd-etcd-client-cert"
46+
47+
// webhook serving cert
48+
SecretWebhook = "karmada-webhook-cert"
49+
)
50+
51+
// PEM key names used inside TLS secrets
52+
const (
53+
KeyTLSCrt = "tls.crt"
54+
KeyTLSKey = "tls.key"
55+
KeyCACrt = "ca.crt"
56+
KeySAPrivate = "sa.key"
57+
KeySAPublic = "sa.pub"
58+
)

‎pkg/karmadactl/cmdinit/cmdinit.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,8 @@ func NewCmdInit(parentCommand string) *cobra.Command {
152152
},
153153
}
154154
flags := cmd.Flags()
155+
// layout of secrets mounted into pods
156+
flags.StringVarP(&opts.SecretLayout, "secret-layout", "", "legacy", "Secret layout mode for generated cert secrets: 'legacy' (single aggregated secret) or 'split' (per-component TLS secrets). Defaults to 'legacy'.")
155157
flags.StringVarP(&opts.ImageRegistry, "private-image-registry", "", "", "Private image registry where pull images from. If set, all required images will be downloaded from it, it would be useful in offline installation scenarios. In addition, you still can use --kube-image-registry to specify the registry for Kubernetes's images.")
156158
flags.StringVarP(&opts.ImagePullPolicy, "image-pull-policy", "", string(corev1.PullIfNotPresent), "The image pull policy for all Karmada components container. One of Always, Never, IfNotPresent. Defaults to IfNotPresent.")
157159
flags.StringSliceVar(&opts.PullSecrets, "image-pull-secrets", nil, "Image pull secrets are used to pull images from the private registry, could be secret list separated by comma (e.g '--image-pull-secrets PullSecret1,PullSecret2', the secrets should be pre-settled in the namespace declared by '--namespace')")

‎pkg/karmadactl/cmdinit/config/types.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,13 @@ type KarmadaInitSpec struct {
7474
// WaitComponentReadyTimeout configures the timeout (in seconds) for waiting for components to be ready
7575
// +optional
7676
WaitComponentReadyTimeout int `json:"waitComponentReadyTimeout,omitempty" yaml:"waitComponentReadyTimeout,omitempty"`
77+
78+
// SecretLayout controls how certificate secrets are organized and mounted during init.
79+
// One of:
80+
// - "legacy": use a single aggregated secret (default behavior prior to split-layout)
81+
// - "split": create per-component TLS secrets (apiserver, etcd client/server, front-proxy, kcm CA/SA, webhook, etc.)
82+
// +optional
83+
SecretLayout string `json:"secretLayout,omitempty" yaml:"secretLayout,omitempty"`
7784
}
7885

7986
// Certificates defines the configuration related to certificates

‎pkg/karmadactl/cmdinit/kubernetes/command.go‎

Lines changed: 38 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -169,25 +169,47 @@ func (i *CommandInitOption) defaultKarmadaAggregatedAPIServerContainerCommand()
169169
if etcdServers = i.ExternalEtcdServers; etcdServers == "" {
170170
etcdServers = strings.TrimRight(i.etcdServers(), ",")
171171
}
172-
command := []string{
173-
"/bin/karmada-aggregated-apiserver",
174-
fmt.Sprintf("--kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, util.KarmadaConfigFieldName)),
175-
fmt.Sprintf("--authentication-kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, util.KarmadaConfigFieldName)),
176-
fmt.Sprintf("--authorization-kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, util.KarmadaConfigFieldName)),
177-
fmt.Sprintf("--etcd-servers=%s", etcdServers),
178-
fmt.Sprintf("--etcd-cafile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdCaCertAndKeyName),
179-
fmt.Sprintf("--etcd-certfile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName),
180-
fmt.Sprintf("--etcd-keyfile=%s/%s.key", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName),
181-
fmt.Sprintf("--tls-cert-file=%s/%s.crt", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName),
182-
fmt.Sprintf("--tls-private-key-file=%s/%s.key", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName),
183-
"--tls-min-version=VersionTLS13",
184-
"--audit-log-path=-",
185-
"--audit-log-maxage=0",
186-
"--audit-log-maxbackup=0",
187-
"--bind-address=$(POD_IP)",
172+
var command []string
173+
if strings.ToLower(i.SecretLayout) == secretLayoutSplit {
174+
command = []string{
175+
"/bin/karmada-aggregated-apiserver",
176+
fmt.Sprintf("--kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, util.KarmadaConfigFieldName)),
177+
fmt.Sprintf("--authentication-kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, util.KarmadaConfigFieldName)),
178+
fmt.Sprintf("--authorization-kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, util.KarmadaConfigFieldName)),
179+
fmt.Sprintf("--etcd-servers=%s", etcdServers),
180+
fmt.Sprintf("--etcd-cafile=%s/ca.crt", etcdClientCertVolumeMountPath),
181+
fmt.Sprintf("--etcd-certfile=%s/tls.crt", etcdClientCertVolumeMountPath),
182+
fmt.Sprintf("--etcd-keyfile=%s/tls.key", etcdClientCertVolumeMountPath),
183+
fmt.Sprintf("--tls-cert-file=%s/tls.crt", serverCertVolumeMountPath),
184+
fmt.Sprintf("--tls-private-key-file=%s/tls.key", serverCertVolumeMountPath),
185+
"--tls-min-version=VersionTLS13",
186+
"--audit-log-path=-",
187+
"--audit-log-maxage=0",
188+
"--audit-log-maxbackup=0",
189+
"--bind-address=$(POD_IP)",
190+
}
191+
} else {
192+
command = []string{
193+
"/bin/karmada-aggregated-apiserver",
194+
fmt.Sprintf("--kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, util.KarmadaConfigFieldName)),
195+
fmt.Sprintf("--authentication-kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, util.KarmadaConfigFieldName)),
196+
fmt.Sprintf("--authorization-kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, util.KarmadaConfigFieldName)),
197+
fmt.Sprintf("--etcd-servers=%s", etcdServers),
198+
fmt.Sprintf("--etcd-cafile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdCaCertAndKeyName),
199+
fmt.Sprintf("--etcd-certfile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName),
200+
fmt.Sprintf("--etcd-keyfile=%s/%s.key", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName),
201+
fmt.Sprintf("--tls-cert-file=%s/%s.crt", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName),
202+
fmt.Sprintf("--tls-private-key-file=%s/%s.key", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName),
203+
"--tls-min-version=VersionTLS13",
204+
"--audit-log-path=-",
205+
"--audit-log-maxage=0",
206+
"--audit-log-maxbackup=0",
207+
"--bind-address=$(POD_IP)",
208+
}
188209
}
189210
if i.ExternalEtcdKeyPrefix != "" {
190211
command = append(command, fmt.Sprintf("--etcd-prefix=%s", i.ExternalEtcdKeyPrefix))
191212
}
213+
192214
return command
193215
}

0 commit comments

Comments
 (0)