Skip to content

Commit 93d9670

Browse files
tiansuo114tiansuo114
committed
feat: karmadactl support split secret layout in init command
Signed-off-by: tiansuo <[email protected]>
1 parent 9928a81 commit 93d9670

File tree

10 files changed

+1071
-265
lines changed

10 files changed

+1071
-265
lines changed

‎.github/workflows/installation-cli.yaml‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,50 @@ jobs:
6363
name: karmadactl_test_logs_${{ matrix.k8s }}
6464
path: ${{ github.workspace }}/karmadactl-test-logs/${{ matrix.k8s }}/
6565

66+
init-with-split-secret:
67+
name: init with split secret
68+
runs-on: ubuntu-22.04
69+
strategy:
70+
fail-fast: false
71+
matrix:
72+
# Latest three minor releases of Kubernetes
73+
k8s: [ v1.31.0, v1.32.0, v1.33.0 ]
74+
steps:
75+
- name: checkout code
76+
uses: actions/checkout@v5
77+
with:
78+
fetch-depth: 0
79+
- name: install Go
80+
uses: actions/setup-go@v6
81+
with:
82+
go-version-file: go.mod
83+
- name: run karmadactl init with config file test
84+
run: |
85+
export CLUSTER_VERSION=kindest/node:${{ matrix.k8s }}
86+
87+
# Run custom test for workload configuration deployment
88+
hack/cli-testing-environment-split-secret.sh
89+
90+
# run a single e2e
91+
export PULL_BASED_CLUSTERS="config-member1:${HOME}/.kube/config-member1.config"
92+
export KUBECONFIG=${HOME}/.kube/karmada-host.config:${HOME}/karmada/karmada-apiserver.config
93+
GO111MODULE=on go install github.com/onsi/ginkgo/v2/ginkgo
94+
ginkgo -v --race --trace -p --focus="[BasicPropagation] propagation testing deployment propagation testing" ./test/e2e/suites/base
95+
- name: export logs for config test
96+
if: always()
97+
run: |
98+
export ARTIFACTS_PATH=${{ github.workspace }}/karmadactl-test-logs/${{ matrix.k8s }}/config
99+
mkdir -p $ARTIFACTS_PATH
100+
101+
mkdir -p $ARTIFACTS_PATH/karmada-host
102+
kind export logs --name=karmada-host $ARTIFACTS_PATH/karmada-host
103+
- name: upload config test logs
104+
if: always()
105+
uses: actions/upload-artifact@v4
106+
with:
107+
name: karmadactl_config_test_logs_${{ matrix.k8s }}
108+
path: ${{ github.workspace }}/karmadactl-test-logs/${{ matrix.k8s }}/config/
109+
66110
init-config:
67111
name: init with config file
68112
runs-on: ubuntu-22.04
@@ -106,3 +150,4 @@ jobs:
106150
with:
107151
name: karmadactl_config_test_logs_${{ matrix.k8s }}
108152
path: ${{ github.workspace }}/karmadactl-test-logs/${{ matrix.k8s }}/config/
153+

‎hack/cli-testing-environment-split-secret.sh‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2023 The Karmada Authors.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
set -o errexit
17+
set -o nounset
18+
set -o pipefail
19+
20+
# This script starts a local karmada control plane with karmadactl and with a certain number of clusters joined.
21+
# This script depends on utils in: ${REPO_ROOT}/hack/util.sh
22+
# 1. used by developer to setup develop environment quickly.
23+
# 2. used by e2e testing to setup test environment automatically.
24+
25+
REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
26+
source "${REPO_ROOT}"/hack/util.sh
27+
28+
# variable define
29+
KUBECONFIG_PATH=${KUBECONFIG_PATH:-"${HOME}/.kube"}
30+
HOST_CLUSTER_NAME=${HOST_CLUSTER_NAME:-"karmada-host"}
31+
MEMBER_CLUSTER_1_NAME=${MEMBER_CLUSTER_1_NAME:-"member1"}
32+
MEMBER_CLUSTER_2_NAME=${MEMBER_CLUSTER_2_NAME:-"member2"}
33+
CLUSTER_VERSION=${CLUSTER_VERSION:-"${DEFAULT_CLUSTER_VERSION}"}
34+
BUILD_PATH=${BUILD_PATH:-"_output/bin/linux/amd64"}
35+
36+
37+
# install kind and kubectl
38+
echo -n "Preparing: 'kind' existence check - "
39+
if util::cmd_exist kind; then
40+
echo "passed"
41+
else
42+
echo "not pass"
43+
# Install kind using the version defined in util.sh
44+
util::install_tools "sigs.k8s.io/kind" "${KIND_VERSION}"
45+
fi
46+
# get arch name and os name in bootstrap
47+
BS_ARCH=$(go env GOARCH)
48+
BS_OS=$(go env GOOS)
49+
# check arch and os name before installing
50+
util::install_environment_check "${BS_ARCH}" "${BS_OS}"
51+
echo -n "Preparing: 'kubectl' existence check - "
52+
if util::cmd_exist kubectl; then
53+
echo "passed"
54+
else
55+
echo "not pass"
56+
util::install_kubectl "" "${BS_ARCH}" "${BS_OS}"
57+
fi
58+
59+
# prepare the newest crds
60+
echo "Prepare the newest crds"
61+
cd charts/karmada/
62+
cp -r _crds crds
63+
tar -zcvf ../../crds.tar.gz crds
64+
cd -
65+
66+
# make images
67+
export VERSION="latest"
68+
export REGISTRY="docker.io/karmada"
69+
make images GOOS="linux" --directory="${REPO_ROOT}"
70+
71+
# make karmadactl binary
72+
make karmadactl
73+
74+
# create host/member1/member2 cluster
75+
echo "Start create clusters..."
76+
hack/create-cluster.sh ${HOST_CLUSTER_NAME} ${KUBECONFIG_PATH}/${HOST_CLUSTER_NAME}.config > /dev/null 2>&1 &
77+
hack/create-cluster.sh ${MEMBER_CLUSTER_1_NAME} ${KUBECONFIG_PATH}/${MEMBER_CLUSTER_1_NAME}.config > /dev/null 2>&1 &
78+
hack/create-cluster.sh ${MEMBER_CLUSTER_2_NAME} ${KUBECONFIG_PATH}/${MEMBER_CLUSTER_2_NAME}.config > /dev/null 2>&1 &
79+
80+
# wait cluster ready
81+
echo "Wait clusters ready..."
82+
util::wait_file_exist ${KUBECONFIG_PATH}/${HOST_CLUSTER_NAME}.config 300
83+
util::wait_context_exist ${HOST_CLUSTER_NAME} ${KUBECONFIG_PATH}/${HOST_CLUSTER_NAME}.config 300
84+
kubectl wait --for=condition=Ready nodes --all --timeout=800s --kubeconfig=${KUBECONFIG_PATH}/${HOST_CLUSTER_NAME}.config
85+
util::wait_nodes_taint_disappear 800 ${KUBECONFIG_PATH}/${HOST_CLUSTER_NAME}.config
86+
87+
util::wait_file_exist ${KUBECONFIG_PATH}/${MEMBER_CLUSTER_1_NAME}.config 300
88+
util::wait_context_exist "${MEMBER_CLUSTER_1_NAME}" ${KUBECONFIG_PATH}/${MEMBER_CLUSTER_1_NAME}.config 300
89+
kubectl wait --for=condition=Ready nodes --all --timeout=800s --kubeconfig=${KUBECONFIG_PATH}/${MEMBER_CLUSTER_1_NAME}.config
90+
util::wait_nodes_taint_disappear 800 ${KUBECONFIG_PATH}/${MEMBER_CLUSTER_1_NAME}.config
91+
92+
util::wait_file_exist ${KUBECONFIG_PATH}/${MEMBER_CLUSTER_2_NAME}.config 300
93+
util::wait_context_exist "${MEMBER_CLUSTER_2_NAME}" ${KUBECONFIG_PATH}/${MEMBER_CLUSTER_2_NAME}.config 300
94+
kubectl wait --for=condition=Ready nodes --all --timeout=800s --kubeconfig=${KUBECONFIG_PATH}/${MEMBER_CLUSTER_2_NAME}.config
95+
util::wait_nodes_taint_disappear 800 ${KUBECONFIG_PATH}/${MEMBER_CLUSTER_2_NAME}.config
96+
97+
# load components images to kind cluster
98+
kind load docker-image "${REGISTRY}/karmada-controller-manager:${VERSION}" --name="${HOST_CLUSTER_NAME}"
99+
kind load docker-image "${REGISTRY}/karmada-scheduler:${VERSION}" --name="${HOST_CLUSTER_NAME}"
100+
kind load docker-image "${REGISTRY}/karmada-webhook:${VERSION}" --name="${HOST_CLUSTER_NAME}"
101+
kind load docker-image "${REGISTRY}/karmada-aggregated-apiserver:${VERSION}" --name="${HOST_CLUSTER_NAME}"
102+
kind load docker-image "${REGISTRY}/karmada-agent:${VERSION}" --name="${MEMBER_CLUSTER_1_NAME}"
103+
104+
# init Karmada control plane
105+
echo "Start init karmada control plane..."
106+
${BUILD_PATH}/karmadactl init --kubeconfig=${KUBECONFIG_PATH}/${HOST_CLUSTER_NAME}.config \
107+
--karmada-controller-manager-image="${REGISTRY}/karmada-controller-manager:${VERSION}" \
108+
--karmada-scheduler-image="${REGISTRY}/karmada-scheduler:${VERSION}" \
109+
--karmada-webhook-image="${REGISTRY}/karmada-webhook:${VERSION}" \
110+
--karmada-aggregated-apiserver-image="${REGISTRY}/karmada-aggregated-apiserver:${VERSION}" \
111+
--karmada-data=${HOME}/karmada \
112+
--karmada-pki=${HOME}/karmada/pki \
113+
--crds=./crds.tar.gz \
114+
--secret-layout='split'
115+
116+
# join cluster
117+
echo "Join member clusters..."
118+
TOKEN_CMD=$(${BUILD_PATH}/karmadactl --kubeconfig ${HOME}/karmada/karmada-apiserver.config token create --print-register-command)
119+
TOKEN=$(echo "$TOKEN_CMD" | grep -o '\--token [^ ]*' | cut -d' ' -f2)
120+
HASH=$(echo "$TOKEN_CMD" | grep -o '\--discovery-token-ca-cert-hash [^ ]*' | cut -d' ' -f2)
121+
ENDPOINT=$(kubectl --kubeconfig ${HOME}/karmada/karmada-apiserver.config config view --minify -o jsonpath='{.clusters[0].cluster.server}' | sed 's|^https://||')
122+
123+
${BUILD_PATH}/karmadactl register ${ENDPOINT} \
124+
--token ${TOKEN} \
125+
--discovery-token-ca-cert-hash ${HASH} \
126+
--kubeconfig=${KUBECONFIG_PATH}/${MEMBER_CLUSTER_1_NAME}.config \
127+
--cluster-name=${MEMBER_CLUSTER_1_NAME} \
128+
--karmada-agent-image "${REGISTRY}/karmada-agent:${VERSION}" \
129+
--v=4
130+
131+
${BUILD_PATH}/karmadactl --kubeconfig ${HOME}/karmada/karmada-apiserver.config join ${MEMBER_CLUSTER_2_NAME} --cluster-kubeconfig=${KUBECONFIG_PATH}/${MEMBER_CLUSTER_2_NAME}.config
132+
133+
kubectl wait --for=condition=Ready clusters --all --timeout=800s --kubeconfig=${HOME}/karmada/karmada-apiserver.config
134+

‎pkg/cert/constants.go‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
/*
2+
Copyright 2025 The Karmada Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package cert
18+
19+
// Package cert centralizes TLS-related constants (secret names and key names)
20+
// so they can be shared by cmdinit and future operator implementations.
21+
22+
// Secret names for split-layout TLS materials
23+
const (
24+
// apiserver
25+
SecretApiserverServer = "karmada-apiserver-cert"
26+
SecretApiserverEtcdClient = "karmada-apiserver-etcd-client-cert"
27+
SecretApiserverFrontProxyClient = "karmada-apiserver-front-proxy-client-cert"
28+
SecretApiserverServiceAccountKeys = "karmada-apiserver-service-account-key-pair"
29+
30+
// aggregated apiserver
31+
SecretAggregatedAPIServerServer = "karmada-aggregated-apiserver-cert"
32+
SecretAggregatedAPIServerEtcdClient = "karmada-aggregated-apiserver-etcd-client-cert"
33+
34+
// kube-controller-manager
35+
SecretKubeControllerManagerCA = "kube-controller-manager-ca-cert"
36+
SecretKubeControllerManagerSAKeys = "kube-controller-manager-service-account-key-pair"
37+
38+
// scheduler(estimator) clients
39+
SecretSchedulerEstimatorClient = "karmada-scheduler-scheduler-estimator-client-cert"
40+
SecretDeschedulerEstimatorClient = "karmada-descheduler-scheduler-estimator-client-cert"
41+
42+
// etcd (internal)
43+
SecretEtcdServer = "etcd-cert"
44+
SecretEtcdClient = "etcd-etcd-client-cert"
45+
46+
// webhook serving cert
47+
SecretWebhook = "karmada-webhook-cert"
48+
)
49+
50+
// PEM key names used inside TLS secrets
51+
const (
52+
KeyTLSCrt = "tls.crt"
53+
KeyTLSKey = "tls.key"
54+
KeyCACrt = "ca.crt"
55+
KeySAPrivate = "sa.key"
56+
KeySAPublic = "sa.pub"
57+
)

‎pkg/karmadactl/cmdinit/cmdinit.go‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -84,8 +84,8 @@ var (
8484

8585
// NewCmdInit install Karmada on Kubernetes
8686
func NewCmdInit(parentCommand string) *cobra.Command {
87-
opts := kubernetes.CommandInitOption{}
88-
cmd := &cobra.Command{
87+
opts := kubernetes.CommandInitOption{}
88+
cmd := &cobra.Command{
8989
Use: "init",
9090
Short: "Install the Karmada control plane in a Kubernetes cluster",
9191
Long: initLong,
@@ -115,9 +115,11 @@ func NewCmdInit(parentCommand string) *cobra.Command {
115115
Annotations: map[string]string{
116116
util.TagCommandGroup: util.GroupClusterRegistration,
117117
},
118-
}
119-
flags := cmd.Flags()
120-
flags.StringVarP(&opts.ImageRegistry, "private-image-registry", "", "", "Private image registry where pull images from. If set, all required images will be downloaded from it, it would be useful in offline installation scenarios. In addition, you still can use --kube-image-registry to specify the registry for Kubernetes's images.")
118+
}
119+
flags := cmd.Flags()
120+
// layout of secrets mounted into pods
121+
flags.StringVarP(&opts.SecretLayout, "secret-layout", "", "legacy", "Secret layout mode for generated cert secrets: 'legacy' (single aggregated secret) or 'split' (per-component TLS secrets). Defaults to 'legacy'.")
122+
flags.StringVarP(&opts.ImageRegistry, "private-image-registry", "", "", "Private image registry where pull images from. If set, all required images will be downloaded from it, it would be useful in offline installation scenarios. In addition, you still can use --kube-image-registry to specify the registry for Kubernetes's images.")
121123
flags.StringVarP(&opts.ImagePullPolicy, "image-pull-policy", "", string(corev1.PullIfNotPresent), "The image pull policy for all Karmada components container. One of Always, Never, IfNotPresent. Defaults to IfNotPresent.")
122124
flags.StringSliceVar(&opts.PullSecrets, "image-pull-secrets", nil, "Image pull secrets are used to pull images from the private registry, could be secret list separated by comma (e.g '--image-pull-secrets PullSecret1,PullSecret2', the secrets should be pre-settled in the namespace declared by '--namespace')")
123125
// kube image registry

‎pkg/karmadactl/cmdinit/config/types.go‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -71,9 +71,16 @@ type KarmadaInitSpec struct {
7171
// +optional
7272
KarmadaPKIPath string `json:"karmadaPKIPath,omitempty" yaml:"karmadaPKIPath,omitempty"`
7373

74-
// WaitComponentReadyTimeout configures the timeout (in seconds) for waiting for components to be ready
75-
// +optional
76-
WaitComponentReadyTimeout int `json:"waitComponentReadyTimeout,omitempty" yaml:"waitComponentReadyTimeout,omitempty"`
74+
// WaitComponentReadyTimeout configures the timeout (in seconds) for waiting for components to be ready
75+
// +optional
76+
WaitComponentReadyTimeout int `json:"waitComponentReadyTimeout,omitempty" yaml:"waitComponentReadyTimeout,omitempty"`
77+
78+
// SecretLayout controls how certificate secrets are organized and mounted during init.
79+
// One of:
80+
// - "legacy": use a single aggregated secret (default behavior prior to split-layout)
81+
// - "split": create per-component TLS secrets (apiserver, etcd client/server, front-proxy, kcm CA/SA, webhook, etc.)
82+
// +optional
83+
SecretLayout string `json:"secretLayout,omitempty" yaml:"secretLayout,omitempty"`
7784
}
7885

7986
// Certificates defines the configuration related to certificates

0 commit comments

Comments
 (0)