diff --git a/.github/workflows/installation-cli.yaml b/.github/workflows/installation-cli.yaml index 11e46a417b0e..ae3f4c67ea59 100644 --- a/.github/workflows/installation-cli.yaml +++ b/.github/workflows/installation-cli.yaml @@ -106,3 +106,47 @@ jobs: with: name: karmadactl_config_test_logs_${{ matrix.k8s }} path: ${{ github.workspace }}/karmadactl-test-logs/${{ matrix.k8s }}/config/ + + test-on-kubernetes-split-secret: + name: Test on Kubernetes (Split Secret) + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + # Latest three minor releases of Kubernetes + k8s: [ v1.31.0, v1.32.0, v1.33.0 ] + steps: + - name: checkout code + uses: actions/checkout@v5 + with: + fetch-depth: 0 + - name: install Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + - name: run karmadactl init test (split secret layout) + run: | + export CLUSTER_VERSION=kindest/node:${{ matrix.k8s }} + + # init e2e environment with split secret layout + hack/cli-testing-environment-split-secret.sh + + # run a single e2e + export PULL_BASED_CLUSTERS="split-secret-member1:${HOME}/.kube/split-secret-member1.config" + export KUBECONFIG=${HOME}/.kube/karmada-host.config:${HOME}/karmada/karmada-apiserver.config + GO111MODULE=on go install github.com/onsi/ginkgo/v2/ginkgo + ginkgo -v --race --trace -p --focus="[BasicPropagation] propagation testing deployment propagation testing" ./test/e2e/suites/base + - name: export logs (split secret) + if: always() + run: | + export ARTIFACTS_PATH=${{ github.workspace }}/karmadactl-test-logs/${{ matrix.k8s }}/split-secret + mkdir -p $ARTIFACTS_PATH + + mkdir -p $ARTIFACTS_PATH/karmada-host + kind export logs --name=karmada-host $ARTIFACTS_PATH/karmada-host + - name: upload logs (split secret) + if: always() + uses: actions/upload-artifact@v4 + with: + name: karmadactl_split_secret_test_logs_${{ matrix.k8s }} + path: ${{ github.workspace }}/karmadactl-test-logs/${{ matrix.k8s }}/split-secret/ diff --git a/hack/cli-testing-environment-split-secret.sh b/hack/cli-testing-environment-split-secret.sh new file mode 100755 index 000000000000..67480cd0aee4 --- /dev/null +++ b/hack/cli-testing-environment-split-secret.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# Copyright 2025 The Karmada Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# This script starts a local karmada control plane with karmadactl and with a certain number of clusters joined, +# identical to hack/cli-testing-environment.sh except adding '--secret-layout=split' for init. +# This script depends on utils in: ${REPO_ROOT}/hack/util.sh +# 1. used by developer to setup develop environment quickly. +# 2. used by e2e testing to setup test environment automatically. + +REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +source "${REPO_ROOT}"/hack/util.sh + +# variable define +KUBECONFIG_PATH=${KUBECONFIG_PATH:-"${HOME}/.kube"} +HOST_CLUSTER_NAME=${HOST_CLUSTER_NAME:-"karmada-host"} +MEMBER_CLUSTER_1_NAME=${MEMBER_CLUSTER_1_NAME:-"split-secret-member1"} +MEMBER_CLUSTER_2_NAME=${MEMBER_CLUSTER_2_NAME:-"split-secret-member2"} +CLUSTER_VERSION=${CLUSTER_VERSION:-"${DEFAULT_CLUSTER_VERSION}"} +BUILD_PATH=${BUILD_PATH:-"_output/bin/linux/amd64"} + +# install kind and kubectl +echo -n "Preparing: 'kind' existence check - " +if util::cmd_exist kind; then + echo "passed" +else + echo "not pass" + # Install kind using the version defined in util.sh + util::install_tools "sigs.k8s.io/kind" "${KIND_VERSION}" +fi +# get arch name and os name in bootstrap +BS_ARCH=$(go env GOARCH) +BS_OS=$(go env GOOS) +# check arch and os name before installing +util::install_environment_check "${BS_ARCH}" "${BS_OS}" +echo -n "Preparing: 'kubectl' existence check - " +if util::cmd_exist kubectl; then + echo "passed" +else + echo "not pass" + util::install_kubectl "" "${BS_ARCH}" "${BS_OS}" +fi + +# prepare the newest crds +echo "Prepare the newest crds" +cd charts/karmada/ +cp -r _crds crds +tar -zcvf ../../crds.tar.gz crds +cd - + +# make images +export VERSION="latest" +export REGISTRY="docker.io/karmada" +make images GOOS="linux" --directory="${REPO_ROOT}" + +# make karmadactl binary +make karmadactl + +# create host/member1/member2 cluster +echo "Start create clusters..." +hack/create-cluster.sh ${HOST_CLUSTER_NAME} ${KUBECONFIG_PATH}/${HOST_CLUSTER_NAME}.config > /dev/null 2>&1 & +hack/create-cluster.sh ${MEMBER_CLUSTER_1_NAME} ${KUBECONFIG_PATH}/${MEMBER_CLUSTER_1_NAME}.config > /dev/null 2>&1 & +hack/create-cluster.sh ${MEMBER_CLUSTER_2_NAME} ${KUBECONFIG_PATH}/${MEMBER_CLUSTER_2_NAME}.config > /dev/null 2>&1 & + +# wait cluster ready +echo "Wait clusters ready..." +util::wait_file_exist ${KUBECONFIG_PATH}/${HOST_CLUSTER_NAME}.config 300 +util::wait_context_exist ${HOST_CLUSTER_NAME} ${KUBECONFIG_PATH}/${HOST_CLUSTER_NAME}.config 300 +kubectl wait --for=condition=Ready nodes --all --timeout=800s --kubeconfig=${KUBECONFIG_PATH}/${HOST_CLUSTER_NAME}.config +util::wait_nodes_taint_disappear 800 ${KUBECONFIG_PATH}/${HOST_CLUSTER_NAME}.config + +util::wait_file_exist ${KUBECONFIG_PATH}/${MEMBER_CLUSTER_1_NAME}.config 300 +util::wait_context_exist "${MEMBER_CLUSTER_1_NAME}" ${KUBECONFIG_PATH}/${MEMBER_CLUSTER_1_NAME}.config 300 +kubectl wait --for=condition=Ready nodes --all --timeout=800s --kubeconfig=${KUBECONFIG_PATH}/${MEMBER_CLUSTER_1_NAME}.config +util::wait_nodes_taint_disappear 800 ${KUBECONFIG_PATH}/${MEMBER_CLUSTER_1_NAME}.config + +util::wait_file_exist ${KUBECONFIG_PATH}/${MEMBER_CLUSTER_2_NAME}.config 300 +util::wait_context_exist "${MEMBER_CLUSTER_2_NAME}" ${KUBECONFIG_PATH}/${MEMBER_CLUSTER_2_NAME}.config 300 +kubectl wait --for=condition=Ready nodes --all --timeout=800s --kubeconfig=${KUBECONFIG_PATH}/${MEMBER_CLUSTER_2_NAME}.config +util::wait_nodes_taint_disappear 800 ${KUBECONFIG_PATH}/${MEMBER_CLUSTER_2_NAME}.config + +# load components images to kind cluster +kind load docker-image "${REGISTRY}/karmada-controller-manager:${VERSION}" --name="${HOST_CLUSTER_NAME}" +kind load docker-image "${REGISTRY}/karmada-scheduler:${VERSION}" --name="${HOST_CLUSTER_NAME}" +kind load docker-image "${REGISTRY}/karmada-webhook:${VERSION}" --name="${HOST_CLUSTER_NAME}" +kind load docker-image "${REGISTRY}/karmada-aggregated-apiserver:${VERSION}" --name="${HOST_CLUSTER_NAME}" +kind load docker-image "${REGISTRY}/karmada-agent:${VERSION}" --name="${MEMBER_CLUSTER_1_NAME}" + +# init Karmada control plane +echo "Start init karmada control plane..." +${BUILD_PATH}/karmadactl init --kubeconfig=${KUBECONFIG_PATH}/${HOST_CLUSTER_NAME}.config \ + --karmada-controller-manager-image="${REGISTRY}/karmada-controller-manager:${VERSION}" \ + --karmada-scheduler-image="${REGISTRY}/karmada-scheduler:${VERSION}" \ + --karmada-webhook-image="${REGISTRY}/karmada-webhook:${VERSION}" \ + --karmada-aggregated-apiserver-image="${REGISTRY}/karmada-aggregated-apiserver:${VERSION}" \ + --karmada-data=${HOME}/karmada \ + --karmada-pki=${HOME}/karmada/pki \ + --crds=./crds.tar.gz \ + --secret-layout='split' + +# join cluster +echo "Join member clusters..." +TOKEN_CMD=$(${BUILD_PATH}/karmadactl --kubeconfig ${HOME}/karmada/karmada-apiserver.config token create --print-register-command) +TOKEN=$(echo "$TOKEN_CMD" | grep -o '\--token [^ ]*' | cut -d' ' -f2) +HASH=$(echo "$TOKEN_CMD" | grep -o '\--discovery-token-ca-cert-hash [^ ]*' | cut -d' ' -f2) +ENDPOINT=$(kubectl --kubeconfig ${HOME}/karmada/karmada-apiserver.config config view --minify -o jsonpath='{.clusters[0].cluster.server}' | sed 's|^https://||') + +${BUILD_PATH}/karmadactl register ${ENDPOINT} \ + --token ${TOKEN} \ + --discovery-token-ca-cert-hash ${HASH} \ + --kubeconfig=${KUBECONFIG_PATH}/${MEMBER_CLUSTER_1_NAME}.config \ + --cluster-name=${MEMBER_CLUSTER_1_NAME} \ + --karmada-agent-image "${REGISTRY}/karmada-agent:${VERSION}" \ + --v=4 + +${BUILD_PATH}/karmadactl --kubeconfig ${HOME}/karmada/karmada-apiserver.config join ${MEMBER_CLUSTER_2_NAME} --cluster-kubeconfig=${KUBECONFIG_PATH}/${MEMBER_CLUSTER_2_NAME}.config + +kubectl wait --for=condition=Ready clusters --all --timeout=800s --kubeconfig=${HOME}/karmada/karmada-apiserver.config diff --git a/pkg/cert/constants.go b/pkg/cert/constants.go new file mode 100644 index 000000000000..a057a643dc81 --- /dev/null +++ b/pkg/cert/constants.go @@ -0,0 +1,58 @@ +/* +Copyright 2025 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cert + +// Package cert centralizes TLS-related constants (secret names and key names) +// so they can be shared by cmdinit and future operator implementations. + +// Secret names for split-layout TLS materials +// nolint:gosec // These are Kubernetes Secret resource names, not credentials. +const ( + // apiserver + SecretApiserverServer = "karmada-apiserver-cert" + SecretApiserverEtcdClient = "karmada-apiserver-etcd-client-cert" + SecretApiserverFrontProxyClient = "karmada-apiserver-front-proxy-client-cert" + SecretApiserverServiceAccountKeys = "karmada-apiserver-service-account-key-pair" + + // aggregated apiserver + SecretAggregatedAPIServerServer = "karmada-aggregated-apiserver-cert" + SecretAggregatedAPIServerEtcdClient = "karmada-aggregated-apiserver-etcd-client-cert" + + // kube-controller-manager + SecretKubeControllerManagerCA = "kube-controller-manager-ca-cert" + SecretKubeControllerManagerSAKeys = "kube-controller-manager-service-account-key-pair" + + // scheduler(estimator) clients + SecretSchedulerEstimatorClient = "karmada-scheduler-scheduler-estimator-client-cert" + SecretDeschedulerEstimatorClient = "karmada-descheduler-scheduler-estimator-client-cert" + + // etcd (internal) + SecretEtcdServer = "etcd-cert" + SecretEtcdClient = "etcd-etcd-client-cert" + + // webhook serving cert + SecretWebhook = "karmada-webhook-cert" +) + +// PEM key names used inside TLS secrets +const ( + KeyTLSCrt = "tls.crt" + KeyTLSKey = "tls.key" + KeyCACrt = "ca.crt" + KeySAPrivate = "sa.key" + KeySAPublic = "sa.pub" +) diff --git a/pkg/karmadactl/cmdinit/cmdinit.go b/pkg/karmadactl/cmdinit/cmdinit.go index 2b65a12e7c69..34576a834c40 100644 --- a/pkg/karmadactl/cmdinit/cmdinit.go +++ b/pkg/karmadactl/cmdinit/cmdinit.go @@ -152,6 +152,8 @@ func NewCmdInit(parentCommand string) *cobra.Command { }, } flags := cmd.Flags() + // layout of secrets mounted into pods + flags.StringVarP(&opts.SecretLayout, "secret-layout", "", "legacy", "Secret layout mode for generated cert secrets: 'legacy' (single aggregated secret) or 'split' (per-component TLS secrets). Defaults to 'legacy'.") flags.StringVarP(&opts.ImageRegistry, "private-image-registry", "", "", "Private image registry where pull images from. If set, all required images will be downloaded from it, it would be useful in offline installation scenarios. In addition, you still can use --kube-image-registry to specify the registry for Kubernetes's images.") flags.StringVarP(&opts.ImagePullPolicy, "image-pull-policy", "", string(corev1.PullIfNotPresent), "The image pull policy for all Karmada components container. One of Always, Never, IfNotPresent. Defaults to IfNotPresent.") flags.StringSliceVar(&opts.PullSecrets, "image-pull-secrets", nil, "Image pull secrets are used to pull images from the private registry, could be secret list separated by comma (e.g '--image-pull-secrets PullSecret1,PullSecret2', the secrets should be pre-settled in the namespace declared by '--namespace')") diff --git a/pkg/karmadactl/cmdinit/config/types.go b/pkg/karmadactl/cmdinit/config/types.go index 0ac760d3231c..e8552e36b84f 100644 --- a/pkg/karmadactl/cmdinit/config/types.go +++ b/pkg/karmadactl/cmdinit/config/types.go @@ -74,6 +74,13 @@ type KarmadaInitSpec struct { // WaitComponentReadyTimeout configures the timeout (in seconds) for waiting for components to be ready // +optional WaitComponentReadyTimeout int `json:"waitComponentReadyTimeout,omitempty" yaml:"waitComponentReadyTimeout,omitempty"` + + // SecretLayout controls how certificate secrets are organized and mounted during init. + // One of: + // - "legacy": use a single aggregated secret (default behavior prior to split-layout) + // - "split": create per-component TLS secrets (apiserver, etcd client/server, front-proxy, kcm CA/SA, webhook, etc.) + // +optional + SecretLayout string `json:"secretLayout,omitempty" yaml:"secretLayout,omitempty"` } // Certificates defines the configuration related to certificates diff --git a/pkg/karmadactl/cmdinit/kubernetes/command.go b/pkg/karmadactl/cmdinit/kubernetes/command.go index 9e7275b8cba0..8c18ec37625e 100644 --- a/pkg/karmadactl/cmdinit/kubernetes/command.go +++ b/pkg/karmadactl/cmdinit/kubernetes/command.go @@ -169,25 +169,47 @@ func (i *CommandInitOption) defaultKarmadaAggregatedAPIServerContainerCommand() if etcdServers = i.ExternalEtcdServers; etcdServers == "" { etcdServers = strings.TrimRight(i.etcdServers(), ",") } - command := []string{ - "/bin/karmada-aggregated-apiserver", - fmt.Sprintf("--kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, util.KarmadaConfigFieldName)), - fmt.Sprintf("--authentication-kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, util.KarmadaConfigFieldName)), - fmt.Sprintf("--authorization-kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, util.KarmadaConfigFieldName)), - fmt.Sprintf("--etcd-servers=%s", etcdServers), - fmt.Sprintf("--etcd-cafile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdCaCertAndKeyName), - fmt.Sprintf("--etcd-certfile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName), - fmt.Sprintf("--etcd-keyfile=%s/%s.key", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName), - fmt.Sprintf("--tls-cert-file=%s/%s.crt", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName), - fmt.Sprintf("--tls-private-key-file=%s/%s.key", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName), - "--tls-min-version=VersionTLS13", - "--audit-log-path=-", - "--audit-log-maxage=0", - "--audit-log-maxbackup=0", - "--bind-address=$(POD_IP)", + var command []string + if strings.ToLower(i.SecretLayout) == secretLayoutSplit { + command = []string{ + "/bin/karmada-aggregated-apiserver", + fmt.Sprintf("--kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, util.KarmadaConfigFieldName)), + fmt.Sprintf("--authentication-kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, util.KarmadaConfigFieldName)), + fmt.Sprintf("--authorization-kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, util.KarmadaConfigFieldName)), + fmt.Sprintf("--etcd-servers=%s", etcdServers), + fmt.Sprintf("--etcd-cafile=%s/ca.crt", etcdClientCertVolumeMountPath), + fmt.Sprintf("--etcd-certfile=%s/tls.crt", etcdClientCertVolumeMountPath), + fmt.Sprintf("--etcd-keyfile=%s/tls.key", etcdClientCertVolumeMountPath), + fmt.Sprintf("--tls-cert-file=%s/tls.crt", serverCertVolumeMountPath), + fmt.Sprintf("--tls-private-key-file=%s/tls.key", serverCertVolumeMountPath), + "--tls-min-version=VersionTLS13", + "--audit-log-path=-", + "--audit-log-maxage=0", + "--audit-log-maxbackup=0", + "--bind-address=$(POD_IP)", + } + } else { + command = []string{ + "/bin/karmada-aggregated-apiserver", + fmt.Sprintf("--kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, util.KarmadaConfigFieldName)), + fmt.Sprintf("--authentication-kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, util.KarmadaConfigFieldName)), + fmt.Sprintf("--authorization-kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, util.KarmadaConfigFieldName)), + fmt.Sprintf("--etcd-servers=%s", etcdServers), + fmt.Sprintf("--etcd-cafile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdCaCertAndKeyName), + fmt.Sprintf("--etcd-certfile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName), + fmt.Sprintf("--etcd-keyfile=%s/%s.key", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName), + fmt.Sprintf("--tls-cert-file=%s/%s.crt", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName), + fmt.Sprintf("--tls-private-key-file=%s/%s.key", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName), + "--tls-min-version=VersionTLS13", + "--audit-log-path=-", + "--audit-log-maxage=0", + "--audit-log-maxbackup=0", + "--bind-address=$(POD_IP)", + } } if i.ExternalEtcdKeyPrefix != "" { command = append(command, fmt.Sprintf("--etcd-prefix=%s", i.ExternalEtcdKeyPrefix)) } + return command } diff --git a/pkg/karmadactl/cmdinit/kubernetes/deploy.go b/pkg/karmadactl/cmdinit/kubernetes/deploy.go index bdb01f5b8a85..6108b6b38a32 100644 --- a/pkg/karmadactl/cmdinit/kubernetes/deploy.go +++ b/pkg/karmadactl/cmdinit/kubernetes/deploy.go @@ -18,7 +18,11 @@ package kubernetes import ( "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" "encoding/base64" + "encoding/pem" "fmt" "net" "os" @@ -36,6 +40,7 @@ import ( "k8s.io/klog/v2" netutils "k8s.io/utils/net" + certconst "github.com/karmada-io/karmada/pkg/cert" "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/cert" initConfig "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/config" "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/karmada" @@ -118,6 +123,10 @@ const ( etcdStorageModePVC = "PVC" etcdStorageModeEmptyDir = "emptyDir" etcdStorageModeHostPath = "hostPath" + + // secret layout values + secretLayoutLegacy = "legacy" + secretLayoutSplit = "split" ) func init() { @@ -229,6 +238,10 @@ type CommandInitOption struct { CaCertFile string CaKeyFile string KarmadaInitFilePath string + + // SecretLayout controls how cert secrets are generated and mounted. + // One of: "legacy" (aggregate secret) or "split" (per-component TLS secrets) + SecretLayout string } func (i *CommandInitOption) validateLocalEtcd(parentCommand string) error { @@ -310,6 +323,10 @@ func (i *CommandInitOption) Validate(parentCommand string) error { return err } + // validate secret layout + if i.SecretLayout != secretLayoutSplit { + i.SecretLayout = secretLayoutLegacy + } if i.KarmadaInitFilePath != "" { cfg, err := initConfig.LoadInitConfiguration(i.KarmadaInitFilePath) if err != nil { @@ -532,6 +549,9 @@ func (i *CommandInitOption) prepareCRD() error { } func (i *CommandInitOption) createCertsSecrets() error { + if strings.ToLower(i.SecretLayout) == secretLayoutSplit { + return i.createSplitCertsSecrets() + } // Create karmada-config Secret karmadaServerURL := fmt.Sprintf("https://%s.%s.svc.%s:%v", karmadaAPIServerDeploymentAndServiceName, i.Namespace, i.HostClusterDomain, karmadaAPIServerContainerPort) config := utils.CreateWithCerts(karmadaServerURL, options.UserName, options.UserName, i.CertAndKeyFileData[fmt.Sprintf("%s.crt", globaloptions.CaCertAndKeyName)], @@ -571,10 +591,10 @@ func (i *CommandInitOption) createCertsSecrets() error { } karmadaWebhookCert := map[string]string{ - "tls.crt": string(i.CertAndKeyFileData[fmt.Sprintf("%s.crt", options.KarmadaCertAndKeyName)]), - "tls.key": string(i.CertAndKeyFileData[fmt.Sprintf("%s.key", options.KarmadaCertAndKeyName)]), + certconst.KeyTLSCrt: string(i.CertAndKeyFileData[fmt.Sprintf("%s.crt", options.KarmadaCertAndKeyName)]), + certconst.KeyTLSKey: string(i.CertAndKeyFileData[fmt.Sprintf("%s.key", options.KarmadaCertAndKeyName)]), } - karmadaWebhookSecret := i.SecretFromSpec(webhookCertsName, corev1.SecretTypeOpaque, karmadaWebhookCert) + karmadaWebhookSecret := i.SecretFromSpec(certconst.SecretWebhook, corev1.SecretTypeOpaque, karmadaWebhookCert) if err := util.CreateOrUpdateSecret(i.KubeClientSet, karmadaWebhookSecret); err != nil { return err } @@ -582,6 +602,201 @@ func (i *CommandInitOption) createCertsSecrets() error { return nil } +// createSplitCertsSecrets creates per-component TLS secrets following the deploy-karmada.sh layout +func (i *CommandInitOption) createSplitCertsSecrets() error { + // Kubeconfig for components (unchanged) + karmadaServerURL := fmt.Sprintf("https://%s.%s.svc.%s:%v", karmadaAPIServerDeploymentAndServiceName, i.Namespace, i.HostClusterDomain, karmadaAPIServerContainerPort) + config := utils.CreateWithCerts(karmadaServerURL, options.UserName, options.UserName, i.CertAndKeyFileData[fmt.Sprintf("%s.crt", globaloptions.CaCertAndKeyName)], + i.CertAndKeyFileData[fmt.Sprintf("%s.key", options.KarmadaCertAndKeyName)], i.CertAndKeyFileData[fmt.Sprintf("%s.crt", options.KarmadaCertAndKeyName)]) + configBytes, err := clientcmd.Write(*config) + if err != nil { + return fmt.Errorf("failure while serializing admin kubeConfig. %v", err) + } + + for _, karmadaConfigSecretName := range karmadaConfigList { + karmadaConfigSecret := i.SecretFromSpec(karmadaConfigSecretName, corev1.SecretTypeOpaque, map[string]string{util.KarmadaConfigFieldName: string(configBytes)}) + if err = util.CreateOrUpdateSecret(i.KubeClientSet, karmadaConfigSecret); err != nil { + return err + } + } + + caCrt := string(i.CertAndKeyFileData[fmt.Sprintf("%s.crt", globaloptions.CaCertAndKeyName)]) + caKey := string(i.CertAndKeyFileData[fmt.Sprintf("%s.key", globaloptions.CaCertAndKeyName)]) + karmadaCrt := string(i.CertAndKeyFileData[fmt.Sprintf("%s.crt", options.KarmadaCertAndKeyName)]) + karmadaKey := string(i.CertAndKeyFileData[fmt.Sprintf("%s.key", options.KarmadaCertAndKeyName)]) + apiserverCrt := string(i.CertAndKeyFileData[fmt.Sprintf("%s.crt", options.ApiserverCertAndKeyName)]) + apiserverKey := string(i.CertAndKeyFileData[fmt.Sprintf("%s.key", options.ApiserverCertAndKeyName)]) + frontProxyCaCrt := string(i.CertAndKeyFileData[fmt.Sprintf("%s.crt", options.FrontProxyCaCertAndKeyName)]) + frontProxyClientCrt := string(i.CertAndKeyFileData[fmt.Sprintf("%s.crt", options.FrontProxyClientCertAndKeyName)]) + frontProxyClientKey := string(i.CertAndKeyFileData[fmt.Sprintf("%s.key", options.FrontProxyClientCertAndKeyName)]) + etcdCaCrt := string(i.CertAndKeyFileData[fmt.Sprintf("%s.crt", options.EtcdCaCertAndKeyName)]) + etcdServerCrt := string(i.CertAndKeyFileData[fmt.Sprintf("%s.crt", options.EtcdServerCertAndKeyName)]) + etcdServerKey := string(i.CertAndKeyFileData[fmt.Sprintf("%s.key", options.EtcdServerCertAndKeyName)]) + etcdClientCrt := string(i.CertAndKeyFileData[fmt.Sprintf("%s.crt", options.EtcdClientCertAndKeyName)]) + etcdClientKey := string(i.CertAndKeyFileData[fmt.Sprintf("%s.key", options.EtcdClientCertAndKeyName)]) + + saPriv, saPub, err := generateServiceAccountKeyPairPEM() + if err != nil { + return fmt.Errorf("generate service account key pair failed: %v", err) + } + saPrivStr := string(saPriv) + saPubStr := string(saPub) + + err = i.createSecret(caCrt, caKey, karmadaCrt, karmadaKey, apiserverCrt, apiserverKey, + frontProxyCaCrt, frontProxyClientCrt, frontProxyClientKey, etcdCaCrt, etcdServerCrt, etcdServerKey, etcdClientCrt, etcdClientKey, saPrivStr, saPubStr) + if err != nil { + return err + } + + // CA-only compatibility secret for addons to read CABundle + compat := i.SecretFromSpec(globaloptions.KarmadaCertsName, corev1.SecretTypeOpaque, map[string]string{certconst.KeyCACrt: caCrt}) + if err := util.CreateOrUpdateSecret(i.KubeClientSet, compat); err != nil { + return err + } + + return nil +} + +func (i *CommandInitOption) createSecret(caCrt, caKey, karmadaCrt, karmadaKey, apiserverCrt, apiserverKey, + frontProxyCaCrt, frontProxyClientCrt, frontProxyClientKey, etcdCaCrt, etcdServerCrt, etcdServerKey, etcdClientCrt, etcdClientKey, saPriv, saPub string) error { + // Delegate secret creation to helper functions + if err := i.createAPIServerSecret(caCrt, apiserverCrt, apiserverKey, etcdCaCrt, etcdClientCrt, etcdClientKey, frontProxyCaCrt, frontProxyClientCrt, frontProxyClientKey, saPriv, saPub); err != nil { + return err + } + if err := i.createAggregatedAPIServerSecret(caCrt, karmadaCrt, karmadaKey, etcdCaCrt, etcdClientCrt, etcdClientKey); err != nil { + return err + } + if err := i.createEtcdSecret(etcdCaCrt, etcdServerCrt, etcdServerKey, etcdClientCrt, etcdClientKey); err != nil { + return err + } + if err := i.createKubeControllerManagerSecret(caCrt, caKey, saPriv, saPub); err != nil { + return err + } + if err := i.createKarmadaWebhookSecret(caCrt, karmadaCrt, karmadaKey); err != nil { + return err + } + if err := i.createKarmadaSchedulerEstimatorSecret(caCrt, karmadaCrt, karmadaKey); err != nil { + return err + } + if err := i.createKarmadaDeschedulerEstimatorSecret(caCrt, karmadaCrt, karmadaKey); err != nil { + return err + } + + return nil +} + +func (i *CommandInitOption) createAPIServerSecret(caCrt, apiserverCrt, apiserverKey, etcdCaCrt, + etcdClientCrt, etcdClientKey, frontProxyCaCrt, frontProxyClientCrt, frontProxyClientKey, saPriv, saPub string) error { + if err := i.createTLSSecret(certconst.SecretApiserverServer, caCrt, apiserverCrt, apiserverKey); err != nil { + return err + } + if err := i.createTLSSecret(certconst.SecretApiserverEtcdClient, etcdCaCrt, etcdClientCrt, etcdClientKey); err != nil { + return err + } + if err := i.createTLSSecret(certconst.SecretApiserverFrontProxyClient, frontProxyCaCrt, frontProxyClientCrt, frontProxyClientKey); err != nil { + return err + } + saSec := i.SecretFromSpec(certconst.SecretApiserverServiceAccountKeys, corev1.SecretTypeOpaque, map[string]string{certconst.KeySAPrivate: saPriv, certconst.KeySAPublic: saPub}) + if err := util.CreateOrUpdateSecret(i.KubeClientSet, saSec); err != nil { + return err + } + + return nil +} + +func (i *CommandInitOption) createAggregatedAPIServerSecret(caCrt, karmadaCrt, karmadaKey, etcdCaCrt, + etcdClientCrt, etcdClientKey string) error { + if err := i.createTLSSecret(certconst.SecretAggregatedAPIServerServer, caCrt, karmadaCrt, karmadaKey); err != nil { + return err + } + if err := i.createTLSSecret(certconst.SecretAggregatedAPIServerEtcdClient, etcdCaCrt, etcdClientCrt, etcdClientKey); err != nil { + return err + } + + return nil +} + +func (i *CommandInitOption) createEtcdSecret(etcdCaCrt, etcdServerCrt, etcdServerKey, + etcdClientCrt, etcdClientKey string) error { + // etcd server & etcd-client (for internal usage like probes/clients) + if !i.isExternalEtcdProvided() { + if err := i.createTLSSecret(certconst.SecretEtcdServer, etcdCaCrt, etcdServerCrt, etcdServerKey); err != nil { + return err + } + if err := i.createTLSSecret(certconst.SecretEtcdClient, etcdCaCrt, etcdClientCrt, etcdClientKey); err != nil { + return err + } + } + + return nil +} + +func (i *CommandInitOption) createKubeControllerManagerSecret(caCrt, caKey, saPriv, saPub string) error { + kcmCA := i.SecretFromSpec(certconst.SecretKubeControllerManagerCA, corev1.SecretTypeTLS, map[string]string{certconst.KeyTLSCrt: caCrt, certconst.KeyTLSKey: caKey}) + if err := util.CreateOrUpdateSecret(i.KubeClientSet, kcmCA); err != nil { + return err + } + kcmSA := i.SecretFromSpec(certconst.SecretKubeControllerManagerSAKeys, corev1.SecretTypeOpaque, map[string]string{certconst.KeySAPrivate: string(saPriv), certconst.KeySAPublic: string(saPub)}) + if err := util.CreateOrUpdateSecret(i.KubeClientSet, kcmSA); err != nil { + return err + } + + return nil +} + +func (i *CommandInitOption) createKarmadaWebhookSecret(caCrt, karmadaCrt, karmadaKey string) error { + if err := i.createTLSSecret(certconst.SecretWebhook, caCrt, karmadaCrt, karmadaKey); err != nil { + return err + } + + return nil +} + +func (i *CommandInitOption) createKarmadaSchedulerEstimatorSecret(caCrt, karmadaCrt, karmadaKey string) error { + if err := i.createTLSSecret(certconst.SecretSchedulerEstimatorClient, caCrt, karmadaCrt, karmadaKey); err != nil { + return err + } + + return nil +} + +func (i *CommandInitOption) createKarmadaDeschedulerEstimatorSecret(caCrt, karmadaCrt, karmadaKey string) error { + if err := i.createTLSSecret(certconst.SecretDeschedulerEstimatorClient, caCrt, karmadaCrt, karmadaKey); err != nil { + return err + } + + return nil +} + +// helper to create TLS secret with optional ca.crt +func (i *CommandInitOption) createTLSSecret(name string, ca, crt, key string) error { + data := map[string]string{certconst.KeyTLSCrt: crt, certconst.KeyTLSKey: key} + if ca != "" { + data[certconst.KeyCACrt] = ca + } + sec := i.SecretFromSpec(name, corev1.SecretTypeTLS, data) + return util.CreateOrUpdateSecret(i.KubeClientSet, sec) +} + +// generateServiceAccountKeyPairPEM returns PEM encoded RSA private key and public key +func generateServiceAccountKeyPairPEM() ([]byte, []byte, error) { + // 3072-bit RSA + priv, err := rsa.GenerateKey(rand.Reader, 3072) + if err != nil { + return nil, nil, err + } + // marshal private key + privDer := x509.MarshalPKCS1PrivateKey(priv) + privPem := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: privDer}) + // marshal public key in PKIX + pubDer, err := x509.MarshalPKIXPublicKey(&priv.PublicKey) + if err != nil { + return nil, nil, err + } + pubPem := pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: pubDer}) + return privPem, pubPem, nil +} + func (i *CommandInitOption) initKarmadaAPIServer() error { if !i.isExternalEtcdProvided() { if err := util.CreateOrUpdateService(i.KubeClientSet, i.makeEtcdService(etcdStatefulSetAndServiceName)); err != nil { @@ -948,6 +1163,7 @@ func (i *CommandInitOption) parseGeneralConfig(spec initConfig.KarmadaInitSpec) i.PullSecrets = spec.Images.ImagePullSecrets } setIfNotZero(&i.WaitComponentReadyTimeout, spec.WaitComponentReadyTimeout) + setIfNotEmpty(&i.SecretLayout, spec.SecretLayout) } // parseCertificateConfig parses certificate-related configuration, including CA files, diff --git a/pkg/karmadactl/cmdinit/kubernetes/deployments.go b/pkg/karmadactl/cmdinit/kubernetes/deployments.go index 97c45eb2aae3..43f7e47307e7 100644 --- a/pkg/karmadactl/cmdinit/kubernetes/deployments.go +++ b/pkg/karmadactl/cmdinit/kubernetes/deployments.go @@ -18,6 +18,8 @@ package kubernetes import ( "fmt" + "path/filepath" + "strings" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -26,6 +28,8 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" + certconst "github.com/karmada-io/karmada/pkg/cert" + "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/options" globaloptions "github.com/karmada-io/karmada/pkg/karmadactl/options" "github.com/karmada-io/karmada/pkg/karmadactl/util" "github.com/karmada-io/karmada/pkg/util/names" @@ -50,7 +54,6 @@ const ( controllerManagerDeploymentAndServiceName = names.KarmadaControllerManagerComponentName controllerManagerSecurePort = 10357 webhookDeploymentAndServiceAccountAndServiceName = names.KarmadaWebhookComponentName - webhookCertsName = "karmada-webhook-cert" webhookCertVolumeMountPath = "/var/serving-cert" webhookPortName = "webhook" webhookTargetPort = 8443 @@ -58,6 +61,23 @@ const ( karmadaAggregatedAPIServerDeploymentAndServiceName = names.KarmadaAggregatedAPIServerComponentName ) +var ( + // split-layout mount paths + serverCertVolumeMountPath = "/etc/karmada/pki/server" + etcdClientCertVolumeMountPath = "/etc/karmada/pki/etcd-client" + frontProxyClientCertVolumeMountPath = "/etc/karmada/pki/front-proxy-client" + saKeyPairVolumeMountPath = "/etc/karmada/pki/service-account-key-pair" + caCertVolumeMountPath = "/etc/karmada/pki/ca" + schedulerEstimatorClientCertVolumeMountPath = "/etc/karmada/pki/scheduler-estimator-client" + // Volume names for split-layout mounts + serverCertVolumeName = "server-cert" + etcdClientCertVolumeName = "etcd-client-cert" + frontProxyClientCertVolumeName = "front-proxy-client-cert" + saKeyPairVolumeName = "service-account-key-pair" + caCertVolumeName = "ca-cert" + schedulerEstimatorClientCertVolumeName = "scheduler-estimator-client-cert" +) + var ( apiServerLabels = map[string]string{"app": karmadaAPIServerDeploymentAndServiceName} kubeControllerManagerLabels = map[string]string{"app": kubeControllerManagerClusterRoleAndDeploymentAndServiceName} @@ -75,6 +95,81 @@ func (i *CommandInitOption) etcdServers() string { return etcdClusterConfig } +func (i *CommandInitOption) karmadaAPIServerContainerCommand() []string { + var etcdServers string + if etcdServers = i.ExternalEtcdServers; etcdServers == "" { + etcdServers = strings.TrimRight(i.etcdServers(), ",") + } + var command []string + if strings.ToLower(i.SecretLayout) == secretLayoutSplit { + command = []string{ + "kube-apiserver", + "--allow-privileged=true", + "--authorization-mode=Node,RBAC", + fmt.Sprintf("--client-ca-file=%s/ca.crt", serverCertVolumeMountPath), + "--enable-bootstrap-token-auth=true", + fmt.Sprintf("--etcd-cafile=%s/ca.crt", etcdClientCertVolumeMountPath), + fmt.Sprintf("--etcd-certfile=%s/tls.crt", etcdClientCertVolumeMountPath), + fmt.Sprintf("--etcd-keyfile=%s/tls.key", etcdClientCertVolumeMountPath), + fmt.Sprintf("--etcd-servers=%s", etcdServers), + "--bind-address=0.0.0.0", + "--disable-admission-plugins=StorageObjectInUseProtection,ServiceAccount", + "--runtime-config=", + fmt.Sprintf("--apiserver-count=%v", i.KarmadaAPIServerReplicas), + fmt.Sprintf("--secure-port=%v", karmadaAPIServerContainerPort), + fmt.Sprintf("--service-account-issuer=https://kubernetes.default.svc.%s", i.HostClusterDomain), + fmt.Sprintf("--service-account-key-file=%s/sa.pub", saKeyPairVolumeMountPath), + fmt.Sprintf("--service-account-signing-key-file=%s/sa.key", saKeyPairVolumeMountPath), + fmt.Sprintf("--service-cluster-ip-range=%s", serviceClusterIP), + fmt.Sprintf("--proxy-client-cert-file=%s/tls.crt", frontProxyClientCertVolumeMountPath), + fmt.Sprintf("--proxy-client-key-file=%s/tls.key", frontProxyClientCertVolumeMountPath), + "--requestheader-allowed-names=front-proxy-client", + fmt.Sprintf("--requestheader-client-ca-file=%s/ca.crt", frontProxyClientCertVolumeMountPath), + "--requestheader-extra-headers-prefix=X-Remote-Extra-", + "--requestheader-group-headers=X-Remote-Group", + "--requestheader-username-headers=X-Remote-User", + fmt.Sprintf("--tls-cert-file=%s/tls.crt", serverCertVolumeMountPath), + fmt.Sprintf("--tls-private-key-file=%s/tls.key", serverCertVolumeMountPath), + "--tls-min-version=VersionTLS13", + } + } else { + command = []string{ + "kube-apiserver", + "--allow-privileged=true", + "--authorization-mode=Node,RBAC", + fmt.Sprintf("--client-ca-file=%s/%s.crt", karmadaCertsVolumeMountPath, globaloptions.CaCertAndKeyName), + "--enable-bootstrap-token-auth=true", + fmt.Sprintf("--etcd-cafile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdCaCertAndKeyName), + fmt.Sprintf("--etcd-certfile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName), + fmt.Sprintf("--etcd-keyfile=%s/%s.key", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName), + fmt.Sprintf("--etcd-servers=%s", etcdServers), + "--bind-address=0.0.0.0", + "--disable-admission-plugins=StorageObjectInUseProtection,ServiceAccount", + "--runtime-config=", + fmt.Sprintf("--apiserver-count=%v", i.KarmadaAPIServerReplicas), + fmt.Sprintf("--secure-port=%v", karmadaAPIServerContainerPort), + fmt.Sprintf("--service-account-issuer=https://kubernetes.default.svc.%s", i.HostClusterDomain), + fmt.Sprintf("--service-account-key-file=%s/%s.key", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName), + fmt.Sprintf("--service-account-signing-key-file=%s/%s.key", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName), + fmt.Sprintf("--service-cluster-ip-range=%s", serviceClusterIP), + fmt.Sprintf("--proxy-client-cert-file=%s/%s.crt", karmadaCertsVolumeMountPath, options.FrontProxyClientCertAndKeyName), + fmt.Sprintf("--proxy-client-key-file=%s/%s.key", karmadaCertsVolumeMountPath, options.FrontProxyClientCertAndKeyName), + "--requestheader-allowed-names=front-proxy-client", + fmt.Sprintf("--requestheader-client-ca-file=%s/%s.crt", karmadaCertsVolumeMountPath, options.FrontProxyCaCertAndKeyName), + "--requestheader-extra-headers-prefix=X-Remote-Extra-", + "--requestheader-group-headers=X-Remote-Group", + "--requestheader-username-headers=X-Remote-User", + fmt.Sprintf("--tls-cert-file=%s/%s.crt", karmadaCertsVolumeMountPath, options.ApiserverCertAndKeyName), + fmt.Sprintf("--tls-private-key-file=%s/%s.key", karmadaCertsVolumeMountPath, options.ApiserverCertAndKeyName), + "--tls-min-version=VersionTLS13", + } + } + if i.ExternalEtcdKeyPrefix != "" { + command = append(command, fmt.Sprintf("--etcd-prefix=%s", i.ExternalEtcdKeyPrefix)) + } + return command +} + func (i *CommandInitOption) makeKarmadaAPIServerDeployment() *appsv1.Deployment { apiServer := &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ @@ -153,27 +248,32 @@ func (i *CommandInitOption) makeKarmadaAPIServerDeployment() *appsv1.Deployment Protocol: corev1.ProtocolTCP, }, }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: globaloptions.KarmadaCertsName, - ReadOnly: true, - MountPath: karmadaCertsVolumeMountPath, - }, - }, + VolumeMounts: func() []corev1.VolumeMount { + if strings.ToLower(i.SecretLayout) == secretLayoutSplit { + return []corev1.VolumeMount{ + {Name: serverCertVolumeName, ReadOnly: true, MountPath: serverCertVolumeMountPath}, + {Name: etcdClientCertVolumeName, ReadOnly: true, MountPath: etcdClientCertVolumeMountPath}, + {Name: frontProxyClientCertVolumeName, ReadOnly: true, MountPath: frontProxyClientCertVolumeMountPath}, + {Name: saKeyPairVolumeName, ReadOnly: true, MountPath: saKeyPairVolumeMountPath}, + } + } + return []corev1.VolumeMount{{Name: globaloptions.KarmadaCertsName, ReadOnly: true, MountPath: karmadaCertsVolumeMountPath}} + }(), LivenessProbe: livenessProbe, ReadinessProbe: readinessProbe, }, }, - Volumes: []corev1.Volume{ - { - Name: globaloptions.KarmadaCertsName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: globaloptions.KarmadaCertsName, - }, - }, - }, - }, + Volumes: func() []corev1.Volume { + if strings.ToLower(i.SecretLayout) == secretLayoutSplit { + return []corev1.Volume{ + {Name: serverCertVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: certconst.SecretApiserverServer}}}, + {Name: etcdClientCertVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: certconst.SecretApiserverEtcdClient}}}, + {Name: frontProxyClientCertVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: certconst.SecretApiserverFrontProxyClient}}}, + {Name: saKeyPairVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: certconst.SecretApiserverServiceAccountKeys}}}, + } + } + return []corev1.Volume{{Name: globaloptions.KarmadaCertsName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: globaloptions.KarmadaCertsName}}}} + }(), //HostNetwork: true, Tolerations: []corev1.Toleration{ { @@ -258,9 +358,46 @@ func (i *CommandInitOption) makeKarmadaKubeControllerManagerDeployment() *appsv1 AutomountServiceAccountToken: ptr.To[bool](false), Containers: []corev1.Container{ { - Name: kubeControllerManagerClusterRoleAndDeploymentAndServiceName, - Image: i.kubeControllerManagerImage(), - Command: i.KubeControllerManagerContainerCmd, + Name: kubeControllerManagerClusterRoleAndDeploymentAndServiceName, + Image: i.kubeControllerManagerImage(), + Command: func() []string { + var clientCAFile, clusterSigningCertFile, clusterSigningKeyFile, rootCAFile, saPrivKeyFile string + if strings.ToLower(i.SecretLayout) == secretLayoutSplit { + clientCAFile = fmt.Sprintf("%s/tls.crt", caCertVolumeMountPath) + clusterSigningCertFile = fmt.Sprintf("%s/tls.crt", caCertVolumeMountPath) + clusterSigningKeyFile = fmt.Sprintf("%s/tls.key", caCertVolumeMountPath) + rootCAFile = fmt.Sprintf("%s/tls.crt", caCertVolumeMountPath) + saPrivKeyFile = fmt.Sprintf("%s/sa.key", saKeyPairVolumeMountPath) + } else { + clientCAFile = fmt.Sprintf("%s/%s.crt", karmadaCertsVolumeMountPath, globaloptions.CaCertAndKeyName) + clusterSigningCertFile = fmt.Sprintf("%s/%s.crt", karmadaCertsVolumeMountPath, globaloptions.CaCertAndKeyName) + clusterSigningKeyFile = fmt.Sprintf("%s/%s.key", karmadaCertsVolumeMountPath, globaloptions.CaCertAndKeyName) + rootCAFile = fmt.Sprintf("%s/%s.crt", karmadaCertsVolumeMountPath, globaloptions.CaCertAndKeyName) + saPrivKeyFile = fmt.Sprintf("%s/%s.key", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName) + } + return []string{ + "kube-controller-manager", + "--allocate-node-cidrs=true", + fmt.Sprintf("--kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, util.KarmadaConfigFieldName)), + fmt.Sprintf("--authentication-kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, util.KarmadaConfigFieldName)), + fmt.Sprintf("--authorization-kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, util.KarmadaConfigFieldName)), + "--bind-address=0.0.0.0", + fmt.Sprintf("--client-ca-file=%s", clientCAFile), + "--cluster-cidr=10.244.0.0/16", + fmt.Sprintf("--cluster-name=%s", options.ClusterName), + fmt.Sprintf("--cluster-signing-cert-file=%s", clusterSigningCertFile), + fmt.Sprintf("--cluster-signing-key-file=%s", clusterSigningKeyFile), + "--controllers=namespace,garbagecollector,serviceaccount-token,ttl-after-finished,bootstrapsigner,tokencleaner,csrcleaner,csrsigning,clusterrole-aggregation", + "--leader-elect=true", + fmt.Sprintf("--leader-elect-resource-namespace=%s", i.Namespace), + "--node-cidr-mask-size=24", + fmt.Sprintf("--root-ca-file=%s", rootCAFile), + fmt.Sprintf("--service-account-private-key-file=%s", saPrivKeyFile), + fmt.Sprintf("--service-cluster-ip-range=%s", serviceClusterIP), + "--use-service-account-credentials=true", + "--v=4", + } + }(), LivenessProbe: livenessProbe, Ports: []corev1.ContainerPort{ { @@ -269,38 +406,34 @@ func (i *CommandInitOption) makeKarmadaKubeControllerManagerDeployment() *appsv1 Protocol: corev1.ProtocolTCP, }, }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: karmadaConfigVolumeName, - ReadOnly: true, - MountPath: karmadaConfigVolumeMountPath, - }, - { - Name: globaloptions.KarmadaCertsName, - ReadOnly: true, - MountPath: karmadaCertsVolumeMountPath, - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: karmadaConfigVolumeName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: util.KarmadaConfigName(names.KubeControllerManagerComponentName), - }, - }, - }, - { - Name: globaloptions.KarmadaCertsName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: globaloptions.KarmadaCertsName, - }, - }, + VolumeMounts: func() []corev1.VolumeMount { + if strings.ToLower(i.SecretLayout) == secretLayoutSplit { + return []corev1.VolumeMount{ + {Name: karmadaConfigVolumeName, ReadOnly: true, MountPath: karmadaConfigVolumeMountPath}, + {Name: caCertVolumeName, ReadOnly: true, MountPath: caCertVolumeMountPath}, + {Name: saKeyPairVolumeName, ReadOnly: true, MountPath: saKeyPairVolumeMountPath}, + } + } + return []corev1.VolumeMount{ + {Name: karmadaConfigVolumeName, ReadOnly: true, MountPath: karmadaConfigVolumeMountPath}, + {Name: globaloptions.KarmadaCertsName, ReadOnly: true, MountPath: karmadaCertsVolumeMountPath}, + } + }(), }, }, + Volumes: func() []corev1.Volume { + if strings.ToLower(i.SecretLayout) == secretLayoutSplit { + return []corev1.Volume{ + {Name: karmadaConfigVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: util.KarmadaConfigName(names.KubeControllerManagerComponentName)}}}, + {Name: caCertVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: certconst.SecretKubeControllerManagerCA}}}, + {Name: saKeyPairVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: certconst.SecretKubeControllerManagerSAKeys}}}, + } + } + return []corev1.Volume{ + {Name: karmadaConfigVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: util.KarmadaConfigName(names.KubeControllerManagerComponentName)}}}, + {Name: globaloptions.KarmadaCertsName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: globaloptions.KarmadaCertsName}}}, + } + }(), Tolerations: []corev1.Toleration{ { Effect: corev1.TaintEffectNoExecute, @@ -395,7 +528,31 @@ func (i *CommandInitOption) makeKarmadaSchedulerDeployment() *appsv1.Deployment }, }, }, - Command: i.KarmadaSchedulerContainerCmd, + Command: func() []string { + var estCAFile, estCertFile, estKeyFile string + if strings.ToLower(i.SecretLayout) == secretLayoutSplit { + estCAFile = fmt.Sprintf("%s/ca.crt", schedulerEstimatorClientCertVolumeMountPath) + estCertFile = fmt.Sprintf("%s/tls.crt", schedulerEstimatorClientCertVolumeMountPath) + estKeyFile = fmt.Sprintf("%s/tls.key", schedulerEstimatorClientCertVolumeMountPath) + } else { + estCAFile = "/etc/karmada/pki/ca.crt" + estCertFile = "/etc/karmada/pki/karmada.crt" + estKeyFile = "/etc/karmada/pki/karmada.key" + } + return []string{ + "/bin/karmada-scheduler", + fmt.Sprintf("--kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, util.KarmadaConfigFieldName)), + "--metrics-bind-address=$(POD_IP):8080", + "--health-probe-bind-address=$(POD_IP):10351", + "--enable-scheduler-estimator=true", + "--leader-elect=true", + fmt.Sprintf("--scheduler-estimator-ca-file=%s", estCAFile), + fmt.Sprintf("--scheduler-estimator-cert-file=%s", estCertFile), + fmt.Sprintf("--scheduler-estimator-key-file=%s", estKeyFile), + fmt.Sprintf("--leader-elect-resource-namespace=%s", i.Namespace), + "--v=4", + } + }(), LivenessProbe: livenessProbe, Ports: []corev1.ContainerPort{ { @@ -404,38 +561,32 @@ func (i *CommandInitOption) makeKarmadaSchedulerDeployment() *appsv1.Deployment Protocol: corev1.ProtocolTCP, }, }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: karmadaConfigVolumeName, - ReadOnly: true, - MountPath: karmadaConfigVolumeMountPath, - }, - { - Name: globaloptions.KarmadaCertsName, - ReadOnly: true, - MountPath: karmadaCertsVolumeMountPath, - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: karmadaConfigVolumeName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: util.KarmadaConfigName(names.KarmadaSchedulerComponentName), - }, - }, - }, - { - Name: globaloptions.KarmadaCertsName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: globaloptions.KarmadaCertsName, - }, - }, + VolumeMounts: func() []corev1.VolumeMount { + if strings.ToLower(i.SecretLayout) == secretLayoutSplit { + return []corev1.VolumeMount{ + {Name: karmadaConfigVolumeName, ReadOnly: true, MountPath: karmadaConfigVolumeMountPath}, + {Name: schedulerEstimatorClientCertVolumeName, ReadOnly: true, MountPath: schedulerEstimatorClientCertVolumeMountPath}, + } + } + return []corev1.VolumeMount{ + {Name: karmadaConfigVolumeName, ReadOnly: true, MountPath: karmadaConfigVolumeMountPath}, + {Name: globaloptions.KarmadaCertsName, ReadOnly: true, MountPath: karmadaCertsVolumeMountPath}, + } + }(), }, }, + Volumes: func() []corev1.Volume { + if strings.ToLower(i.SecretLayout) == secretLayoutSplit { + return []corev1.Volume{ + {Name: karmadaConfigVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: util.KarmadaConfigName(names.KarmadaSchedulerComponentName)}}}, + {Name: schedulerEstimatorClientCertVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: certconst.SecretSchedulerEstimatorClient}}}, + } + } + return []corev1.Volume{ + {Name: karmadaConfigVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: util.KarmadaConfigName(names.KarmadaSchedulerComponentName)}}}, + {Name: globaloptions.KarmadaCertsName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: globaloptions.KarmadaCertsName}}}, + } + }(), Tolerations: []corev1.Toleration{ { Effect: corev1.TaintEffectNoExecute, @@ -658,7 +809,24 @@ func (i *CommandInitOption) makeKarmadaWebhookDeployment() *appsv1.Deployment { }, }, }, - Command: i.KarmadaWebhookContainerCmd, + Command: func() []string { + var certDir string + if strings.ToLower(i.SecretLayout) == secretLayoutSplit { + certDir = serverCertVolumeMountPath + } else { + certDir = webhookCertVolumeMountPath + } + return []string{ + "/bin/karmada-webhook", + fmt.Sprintf("--kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, util.KarmadaConfigFieldName)), + "--bind-address=$(POD_IP)", + "--metrics-bind-address=$(POD_IP):8080", + "--health-probe-bind-address=$(POD_IP):8000", + fmt.Sprintf("--secure-port=%v", webhookTargetPort), + fmt.Sprintf("--cert-dir=%s", certDir), + "--v=4", + } + }(), Ports: []corev1.ContainerPort{ { Name: webhookPortName, @@ -671,39 +839,33 @@ func (i *CommandInitOption) makeKarmadaWebhookDeployment() *appsv1.Deployment { Protocol: corev1.ProtocolTCP, }, }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: karmadaConfigVolumeName, - ReadOnly: true, - MountPath: karmadaConfigVolumeMountPath, - }, - { - Name: webhookCertsName, - ReadOnly: true, - MountPath: webhookCertVolumeMountPath, - }, - }, + VolumeMounts: func() []corev1.VolumeMount { + if strings.ToLower(i.SecretLayout) == secretLayoutSplit { + return []corev1.VolumeMount{ + {Name: karmadaConfigVolumeName, ReadOnly: true, MountPath: karmadaConfigVolumeMountPath}, + {Name: serverCertVolumeName, ReadOnly: true, MountPath: serverCertVolumeMountPath}, + } + } + return []corev1.VolumeMount{ + {Name: karmadaConfigVolumeName, ReadOnly: true, MountPath: karmadaConfigVolumeMountPath}, + {Name: certconst.SecretWebhook, ReadOnly: true, MountPath: webhookCertVolumeMountPath}, + } + }(), ReadinessProbe: readinesProbe, }, }, - Volumes: []corev1.Volume{ - { - Name: karmadaConfigVolumeName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: util.KarmadaConfigName(names.KarmadaWebhookComponentName), - }, - }, - }, - { - Name: webhookCertsName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: webhookCertsName, - }, - }, - }, - }, + Volumes: func() []corev1.Volume { + if strings.ToLower(i.SecretLayout) == secretLayoutSplit { + return []corev1.Volume{ + {Name: karmadaConfigVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: util.KarmadaConfigName(names.KarmadaWebhookComponentName)}}}, + {Name: serverCertVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: certconst.SecretWebhook}}}, + } + } + return []corev1.Volume{ + {Name: karmadaConfigVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: util.KarmadaConfigName(names.KarmadaWebhookComponentName)}}}, + {Name: certconst.SecretWebhook, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: certconst.SecretWebhook}}}, + } + }(), Tolerations: []corev1.Toleration{ { Effect: corev1.TaintEffectNoExecute, @@ -776,6 +938,10 @@ func (i *CommandInitOption) makeKarmadaAggregatedAPIServerDeployment() *appsv1.D TimeoutSeconds: 15, } + var etcdServers string + if etcdServers = i.ExternalEtcdServers; etcdServers == "" { + etcdServers = strings.TrimRight(i.etcdServers(), ",") + } podSpec := corev1.PodSpec{ ImagePullSecrets: i.getImagePullSecrets(), PriorityClassName: i.KarmadaAggregatedAPIServerPriorityClass, @@ -821,38 +987,34 @@ func (i *CommandInitOption) makeKarmadaAggregatedAPIServerDeployment() *appsv1.D corev1.ResourceCPU: resource.MustParse("100m"), }, }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: karmadaConfigVolumeName, - ReadOnly: true, - MountPath: karmadaConfigVolumeMountPath, - }, - { - Name: globaloptions.KarmadaCertsName, - ReadOnly: true, - MountPath: karmadaCertsVolumeMountPath, - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: karmadaConfigVolumeName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: util.KarmadaConfigName(names.KarmadaAggregatedAPIServerComponentName), - }, - }, - }, - { - Name: globaloptions.KarmadaCertsName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: globaloptions.KarmadaCertsName, - }, - }, + VolumeMounts: func() []corev1.VolumeMount { + if strings.ToLower(i.SecretLayout) == secretLayoutSplit { + return []corev1.VolumeMount{ + {Name: karmadaConfigVolumeName, ReadOnly: true, MountPath: karmadaConfigVolumeMountPath}, + {Name: serverCertVolumeName, ReadOnly: true, MountPath: serverCertVolumeMountPath}, + {Name: etcdClientCertVolumeName, ReadOnly: true, MountPath: etcdClientCertVolumeMountPath}, + } + } + return []corev1.VolumeMount{ + {Name: karmadaConfigVolumeName, ReadOnly: true, MountPath: karmadaConfigVolumeMountPath}, + {Name: globaloptions.KarmadaCertsName, ReadOnly: true, MountPath: karmadaCertsVolumeMountPath}, + } + }(), }, }, + Volumes: func() []corev1.Volume { + if strings.ToLower(i.SecretLayout) == secretLayoutSplit { + return []corev1.Volume{ + {Name: karmadaConfigVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: util.KarmadaConfigName(names.KarmadaAggregatedAPIServerComponentName)}}}, + {Name: serverCertVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: certconst.SecretAggregatedAPIServerServer}}}, + {Name: etcdClientCertVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: certconst.SecretAggregatedAPIServerEtcdClient}}}, + } + } + return []corev1.Volume{ + {Name: karmadaConfigVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: util.KarmadaConfigName(names.KarmadaAggregatedAPIServerComponentName)}}}, + {Name: globaloptions.KarmadaCertsName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: globaloptions.KarmadaCertsName}}}, + } + }(), Tolerations: []corev1.Toleration{ { Effect: corev1.TaintEffectNoExecute, diff --git a/pkg/karmadactl/cmdinit/kubernetes/deployments_test.go b/pkg/karmadactl/cmdinit/kubernetes/deployments_test.go index 97670a1780d8..6255abfecb45 100644 --- a/pkg/karmadactl/cmdinit/kubernetes/deployments_test.go +++ b/pkg/karmadactl/cmdinit/kubernetes/deployments_test.go @@ -17,7 +17,18 @@ limitations under the License. package kubernetes import ( + "context" + "fmt" "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + certconst "github.com/karmada-io/karmada/pkg/cert" + initopt "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/options" + globalopt "github.com/karmada-io/karmada/pkg/karmadactl/options" + "github.com/karmada-io/karmada/pkg/karmadactl/util" + "github.com/karmada-io/karmada/pkg/util/names" ) func TestCommandInitOption_etcdServers(t *testing.T) { @@ -82,3 +93,173 @@ func TestCommandInitOption_makeKarmadaAggregatedAPIServerDeployment(t *testing.T t.Error("CommandInitOption.makeKarmadaAggregatedAPIServerDeployment() returns nil") } } + +// helpers +func hasFlag(cmd []string, want string) bool { + for _, c := range cmd { + if c == want { + return true + } + } + return false +} + +func TestKarmadaAPIServerContainerCommand_Flags_SplitAndLegacy(t *testing.T) { + // split + split := CommandInitOption{ + SecretLayout: "split", + Namespace: "karmada", + EtcdReplicas: 1, + } + scmd := split.karmadaAPIServerContainerCommand() + if !hasFlag(scmd, fmt.Sprintf("--client-ca-file=%s/ca.crt", serverCertVolumeMountPath)) { + t.Fatalf("split: missing --client-ca-file=%s/ca.crt", serverCertVolumeMountPath) + } + if !hasFlag(scmd, fmt.Sprintf("--etcd-cafile=%s/ca.crt", etcdClientCertVolumeMountPath)) || + !hasFlag(scmd, fmt.Sprintf("--etcd-certfile=%s/tls.crt", etcdClientCertVolumeMountPath)) || + !hasFlag(scmd, fmt.Sprintf("--etcd-keyfile=%s/tls.key", etcdClientCertVolumeMountPath)) { + t.Fatalf("split: etcd client flags not using etcd-client mount path") + } + if !hasFlag(scmd, fmt.Sprintf("--tls-cert-file=%s/tls.crt", serverCertVolumeMountPath)) || + !hasFlag(scmd, fmt.Sprintf("--tls-private-key-file=%s/tls.key", serverCertVolumeMountPath)) { + t.Fatalf("split: server tls flags not using server mount path") + } + + // legacy + legacy := CommandInitOption{ + Namespace: "karmada", + EtcdReplicas: 1, + } + lcmd := legacy.karmadaAPIServerContainerCommand() + if !hasFlag(lcmd, fmt.Sprintf("--client-ca-file=%s/%s.crt", karmadaCertsVolumeMountPath, globalopt.CaCertAndKeyName)) { + t.Fatalf("legacy: missing client-ca-file from karmada-cert") + } + if !hasFlag(lcmd, fmt.Sprintf("--etcd-cafile=%s/%s.crt", karmadaCertsVolumeMountPath, initopt.EtcdCaCertAndKeyName)) || + !hasFlag(lcmd, fmt.Sprintf("--etcd-certfile=%s/%s.crt", karmadaCertsVolumeMountPath, initopt.EtcdClientCertAndKeyName)) || + !hasFlag(lcmd, fmt.Sprintf("--etcd-keyfile=%s/%s.key", karmadaCertsVolumeMountPath, initopt.EtcdClientCertAndKeyName)) { + t.Fatalf("legacy: missing etcd client flags from karmada-cert") + } +} + +func TestMakeKarmadaAPIServerDeployment_SplitVolumesAndSecrets(t *testing.T) { + opt := CommandInitOption{Namespace: "karmada", SecretLayout: "split"} + dep := opt.makeKarmadaAPIServerDeployment() + ps := dep.Spec.Template.Spec + + // volume mounts present with expected mount paths + wantMounts := map[string]string{ + serverCertVolumeName: serverCertVolumeMountPath, + etcdClientCertVolumeName: etcdClientCertVolumeMountPath, + frontProxyClientCertVolumeName: frontProxyClientCertVolumeMountPath, + saKeyPairVolumeName: saKeyPairVolumeMountPath, + } + for name, path := range wantMounts { + found := false + for _, m := range ps.Containers[0].VolumeMounts { + if m.Name == name && m.MountPath == path { + found = true + break + } + } + if !found { + t.Fatalf("split: missing VolumeMount %q at %q", name, path) + } + } + + // volumes should reference expected secret names + wantSecrets := map[string]string{ + serverCertVolumeName: certconst.SecretApiserverServer, + etcdClientCertVolumeName: certconst.SecretApiserverEtcdClient, + frontProxyClientCertVolumeName: certconst.SecretApiserverFrontProxyClient, + saKeyPairVolumeName: certconst.SecretApiserverServiceAccountKeys, + } + for vname, sname := range wantSecrets { + found := false + for _, v := range ps.Volumes { + if v.Name == vname && v.VolumeSource.Secret != nil && v.VolumeSource.Secret.SecretName == sname { + found = true + break + } + } + if !found { + t.Fatalf("split: volume %q missing or secretName != %q", vname, sname) + } + } +} + +func TestCreateSplitCertsSecrets_CreatesExpectedSecrets(t *testing.T) { + opt := CommandInitOption{ + Namespace: "karmada", + KubeClientSet: fake.NewClientset(), + } + // Minimal fake data for required certs/keys + opt.CertAndKeyFileData = map[string][]byte{ + fmt.Sprintf("%s.crt", globalopt.CaCertAndKeyName): []byte("CA"), + fmt.Sprintf("%s.key", globalopt.CaCertAndKeyName): []byte("CAK"), + fmt.Sprintf("%s.crt", initopt.KarmadaCertAndKeyName): []byte("KAR"), + fmt.Sprintf("%s.key", initopt.KarmadaCertAndKeyName): []byte("KARK"), + fmt.Sprintf("%s.crt", initopt.ApiserverCertAndKeyName): []byte("APS"), + fmt.Sprintf("%s.key", initopt.ApiserverCertAndKeyName): []byte("APSK"), + fmt.Sprintf("%s.crt", initopt.FrontProxyCaCertAndKeyName): []byte("FPCA"), + fmt.Sprintf("%s.crt", initopt.FrontProxyClientCertAndKeyName): []byte("FPC"), + fmt.Sprintf("%s.key", initopt.FrontProxyClientCertAndKeyName): []byte("FPCK"), + fmt.Sprintf("%s.crt", initopt.EtcdCaCertAndKeyName): []byte("ECA"), + fmt.Sprintf("%s.crt", initopt.EtcdServerCertAndKeyName): []byte("ESC"), + fmt.Sprintf("%s.key", initopt.EtcdServerCertAndKeyName): []byte("ESK"), + fmt.Sprintf("%s.crt", initopt.EtcdClientCertAndKeyName): []byte("ECC"), + fmt.Sprintf("%s.key", initopt.EtcdClientCertAndKeyName): []byte("ECK"), + } + + if err := opt.createSplitCertsSecrets(); err != nil { + t.Fatalf("createSplitCertsSecrets error: %v", err) + } + + // pick representative secrets to assert + secretNames := []string{ + certconst.SecretApiserverServer, + certconst.SecretApiserverEtcdClient, + certconst.SecretAggregatedAPIServerServer, + certconst.SecretEtcdServer, + certconst.SecretKubeControllerManagerCA, + certconst.SecretWebhook, + certconst.SecretSchedulerEstimatorClient, + globalopt.KarmadaCertsName, // CA-only compat + } + for _, n := range secretNames { + s, err := opt.KubeClientSet.CoreV1().Secrets(opt.Namespace).Get(context.Background(), n, metav1.GetOptions{}) + if err != nil { + t.Fatalf("expected secret %q to exist: %v", n, err) + } + switch n { + case globalopt.KarmadaCertsName: + if _, ok := s.StringData[certconst.KeyCACrt]; !ok { + t.Fatalf("compat secret %q missing %q", n, certconst.KeyCACrt) + } + case certconst.SecretApiserverServiceAccountKeys, certconst.SecretKubeControllerManagerSAKeys: + if _, ok := s.StringData[certconst.KeySAPrivate]; !ok { + t.Fatalf("secret %q missing %q", n, certconst.KeySAPrivate) + } + if _, ok := s.StringData[certconst.KeySAPublic]; !ok { + t.Fatalf("secret %q missing %q", n, certconst.KeySAPublic) + } + default: + if _, ok := s.StringData[certconst.KeyTLSCrt]; !ok { + t.Fatalf("secret %q missing %q", n, certconst.KeyTLSCrt) + } + if _, ok := s.StringData[certconst.KeyTLSKey]; !ok { + t.Fatalf("secret %q missing %q", n, certconst.KeyTLSKey) + } + } + } + + // also ensure a few config secrets exist (created from karmadaConfigList) + confNames := []string{ + util.KarmadaConfigName(names.KarmadaAggregatedAPIServerComponentName), + util.KarmadaConfigName(names.KubeControllerManagerComponentName), + } + for _, cn := range confNames { + if _, err := opt.KubeClientSet.CoreV1().Secrets(opt.Namespace).Get(context.Background(), cn, metav1.GetOptions{}); err != nil { + t.Fatalf("expected config secret %q to exist: %v", cn, err) + } + } +} diff --git a/pkg/karmadactl/cmdinit/kubernetes/statefulset.go b/pkg/karmadactl/cmdinit/kubernetes/statefulset.go index dc846e610716..2e935263a59c 100644 --- a/pkg/karmadactl/cmdinit/kubernetes/statefulset.go +++ b/pkg/karmadactl/cmdinit/kubernetes/statefulset.go @@ -26,6 +26,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/component-base/cli/flag" "k8s.io/utils/ptr" + + certconst "github.com/karmada-io/karmada/pkg/cert" + "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/options" ) const ( @@ -56,14 +59,22 @@ var ( func (i *CommandInitOption) etcdVolume() (*[]corev1.Volume, *corev1.PersistentVolumeClaim) { var Volumes []corev1.Volume - - secretVolume := corev1.Volume{ - Name: etcdCertName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: etcdCertName, + if strings.ToLower(i.SecretLayout) == secretLayoutSplit { + // split layout: mount server cert and etcd client cert separately + Volumes = append(Volumes, + corev1.Volume{Name: serverCertVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: certconst.SecretEtcdServer}}}, + corev1.Volume{Name: etcdClientCertVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: certconst.SecretEtcdClient}}}, + ) + } else { + secretVolume := corev1.Volume{ + Name: etcdCertName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: etcdCertName, + }, }, - }, + } + Volumes = append(Volumes, secretVolume) } configVolume := corev1.Volume{ Name: etcdContainerConfigVolumeMountName, @@ -71,7 +82,7 @@ func (i *CommandInitOption) etcdVolume() (*[]corev1.Volume, *corev1.PersistentVo EmptyDir: &corev1.EmptyDirVolumeSource{}, }, } - Volumes = append(Volumes, secretVolume, configVolume) + Volumes = append(Volumes, configVolume) switch i.EtcdStorageMode { case etcdStorageModePVC: @@ -125,6 +136,116 @@ func (i *CommandInitOption) etcdVolume() (*[]corev1.Volume, *corev1.PersistentVo } } +func (i *CommandInitOption) etcdInitContainerCommand() []string { + etcdClusterConfig := "" + for v := int32(0); v < i.EtcdReplicas; v++ { + etcdClusterConfig += fmt.Sprintf("%s-%v=http://%s-%v.%s.%s.svc.%s:%v", etcdStatefulSetAndServiceName, v, etcdStatefulSetAndServiceName, v, etcdStatefulSetAndServiceName, i.Namespace, i.HostClusterDomain, etcdContainerServerPort) + "," + } + + command := []string{ + "sh", + "-c", + fmt.Sprintf( + `set -ex +cat <], got %#v", got) + } + script := got[2] + for _, want := range []string{ + "name: ${POD_NAME}", + "client-transport-security:", + "peer-transport-security:", + "trusted-ca-file: " + etcdClientCertVolumeMountPath + "/ca.crt", + "cert-file: " + serverCertVolumeMountPath + "/tls.crt", + "key-file: " + serverCertVolumeMountPath + "/tls.key", + } { + if !strings.Contains(script, want) { + t.Fatalf("script missing %q", want) + } + } +} + +func TestEtcdVolume_SplitVolumesAndSecrets(t *testing.T) { + opt := CommandInitOption{Namespace: "karmada", SecretLayout: "split"} + vols, _ := opt.etcdVolume() + if vols == nil { + t.Fatalf("vols nil") + } + var haveServer, haveClient, haveConfig bool + for _, v := range *vols { + switch v.Name { + case serverCertVolumeName: + haveServer = v.VolumeSource.Secret != nil && v.VolumeSource.Secret.SecretName == certconst.SecretEtcdServer + case etcdClientCertVolumeName: + haveClient = v.VolumeSource.Secret != nil && v.VolumeSource.Secret.SecretName == certconst.SecretEtcdClient + case etcdContainerConfigVolumeMountName: + haveConfig = v.VolumeSource.EmptyDir != nil + } + } + if !haveServer || !haveClient || !haveConfig { + t.Fatalf("split etcd volumes not as expected: server=%v client=%v config=%v", haveServer, haveClient, haveConfig) + } +} + func TestCommandInitIOption_makeETCDStatefulSet(t *testing.T) { tests := []struct { name string