Add JOYLIVE_CONTROL_PLANE_URL environment variable and refactor config package

Author: chenzhiguo

deploy/all-cr.yaml

@@ -741,6 +741,8 @@ spec:
               value: joylive
             - name: JOYLIVE_MATCH_ENV_LABELS
               value: x-live, x-service
+            - name: JOYLIVE_CONTROL_PLANE_URL
+              value: 
           name: joylive-injector
           image: ghcr.m.daocloud.io/jd-opensource/joylive-injector:v1.2.2
           imagePullPolicy: Always

deploy/joylive-injector/templates/deployment.yaml

@@ -34,6 +34,8 @@ spec:
               value: {{ .Values.namespace }}
             - name: JOYLIVE_MATCH_ENV_LABELS
               value: {{ .Values.matchEnvLabels }}
+            - name: JOYLIVE_CONTROL_PLANE_URL
+              value: {{ .Values.controlPlaneUrl }}
           name: joylive-injector
           image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
           imagePullPolicy: {{ .Values.image.pullPolicy }}

deploy/joylive-injector/values.yaml

@@ -25,6 +25,7 @@ containerArgs:
   - --key=/etc/kubernetes/ssl/dac-key.pem
 
 matchEnvLabels: x-live, x-service
+controlPlaneUrl:
 
 imagePullSecrets: [ ]
 nameOverride: ""

deploy/packages/joylive-injector-1.3.7.tgz

@@ -16,6 +16,7 @@ import (
 
 	"github.com/jd-opensource/joylive-injector/pkg/admission"
 	_ "github.com/jd-opensource/joylive-injector/pkg/mutation"
+	_ "github.com/jd-opensource/joylive-injector/pkg/watcher"
 
 	"github.com/jd-opensource/joylive-injector/pkg/config"
 

main.go

@@ -5,32 +5,34 @@ import (
 
 	v1 "github.com/jd-opensource/joylive-injector/client-go/apis/injector/v1"
 	"github.com/jd-opensource/joylive-injector/pkg/log"
-	"github.com/jd-opensource/joylive-injector/pkg/resource"
-	"go.uber.org/zap"
 )
 
 const (
-	AgentConfig           = "config.yaml"
-	InjectorConfigName    = "injector.yaml"
-	LogConfig             = "logback.xml"
-	BootConfig            = "bootstrap.properties"
-	ConfigMountPath       = "/joylive/config"
-	EmptyDirMountPath     = "/joylive"
-	InitEmptyDirMountPath = "/agent"
-	InitContainerCmd      = "/bin/sh"
-	InitContainerArgs     = "-c, cp -r /joylive/* /agent && chmod -R 777 /agent"
-	ConfigMapEnvName      = "JOYLIVE_CONFIGMAP_NAME"
-	NamespaceEnvName      = "JOYLIVE_NAMESPACE"
-	MatchLabelsEnvName    = "JOYLIVE_MATCH_ENV_LABELS"
-	DefaultNamespace      = "joylive"
-	AgentVersionLabel     = "x-live-version"
+	AgentConfig            = "config.yaml"
+	InjectorConfigName     = "injector.yaml"
+	LogConfig              = "logback.xml"
+	BootConfig             = "bootstrap.properties"
+	ConfigMountPath        = "/joylive/config"
+	EmptyDirMountPath      = "/joylive"
+	InitEmptyDirMountPath  = "/agent"
+	InitContainerCmd       = "/bin/sh"
+	InitContainerArgs      = "-c, cp -r /joylive/* /agent && chmod -R 777 /agent"
+	ConfigMapEnvName       = "JOYLIVE_CONFIGMAP_NAME"
+	NamespaceEnvName       = "JOYLIVE_NAMESPACE"
+	MatchLabelsEnvName     = "JOYLIVE_MATCH_ENV_LABELS"
+	ControlPlaneUrlEnvName = "JOYLIVE_CONTROL_PLANE_URL"
+	DefaultNamespace       = "joylive"
+	AgentVersionLabel      = "x-live-version"
+	ServiceSpaceLabel      = "jmsf.jd.com/space"
+	ApplicationLabel       = "jmsf.jd.com/application"
 )
 
 var (
-	Cert        string
-	Key         string
-	Addr        string
-	MatchLabels string
+	Cert            string
+	Key             string
+	Addr            string
+	MatchLabels     string
+	ControlPlaneUrl string
 )
 
 // injection_deploy config
@@ -47,30 +49,10 @@ var (
 )
 
 func init() {
-	// Start the ConfigMap listener and initialize the content
-	cmWatcher := NewConfigMapWatcher(resource.GetResource().ClientSet)
-	err := cmWatcher.Start()
-	if err != nil {
-		log.Fatal("start cmWatcher error", zap.Error(err))
-	}
-	err = cmWatcher.InitConfigMap(GetNamespace())
-	if err != nil {
-		log.Fatal("init cm error", zap.Error(err))
-	}
-
-	// Start the AgentVersion listener and initialize the content
-	avWatcher := NewAgentVersionWatcher(resource.GetResource().RestConfig)
-	err = avWatcher.Start()
-	if err != nil {
-		log.Fatal("start avWatcher error", zap.Error(err))
-	}
-	err = avWatcher.InitAgentVersion(GetNamespace())
-	if err != nil {
-		log.Fatal("init agentVersion error", zap.Error(err))
-	}
-
 	// Initialize the default matchLabels from environment variables
 	MatchLabels = os.Getenv(MatchLabelsEnvName)
+	// Initialize the default control plane url from environment variables
+	ControlPlaneUrl = os.Getenv(ControlPlaneUrlEnvName)
 }
 
 func GetNamespace() string {

pkg/config/config.go

@@ -68,7 +68,7 @@ func injectionDeploy(request *admissionv1.AdmissionRequest) (*admissionv1.Admiss
 			}, nil
 		}
 		log.Infof("[mutation] /injection-deploy: create config map for this deployment: %s, namespace: %s", deploy.Name, deploy.Namespace)
-		err = createConfigMap(&deploy)
+		err = createOrUpdateConfigMap(&deploy)
 		if err != nil {
 			errMsg := fmt.Sprintf("[mutation] /injection-deploy: failed to create configmap: %v", err)
 			log.Error(errMsg)
@@ -80,6 +80,32 @@ func injectionDeploy(request *admissionv1.AdmissionRequest) (*admissionv1.Admiss
 				},
 			}, nil
 		}
+		// Add environment variables to the deployment
+		if len(config.ControlPlaneUrl) > 0 {
+			// Get the application environment variables
+			labels := deploy.GetLabels()
+			serviceSpace, application := labels[config.ServiceSpaceLabel], labels[config.ApplicationLabel]
+			if len(serviceSpace) > 0 && len(application) > 0 {
+				envs, err := resource.GetApplicationEnvironments(labels[config.ServiceSpaceLabel], labels[config.ApplicationLabel])
+				if err != nil {
+					errMsg := fmt.Sprintf("[mutation] /injection-deploy: failed to get application environments: %v", err)
+					log.Error(errMsg)
+					return &admissionv1.AdmissionResponse{
+						Allowed: false,
+						Result: &metav1.Status{
+							Code:    http.StatusInternalServerError,
+							Message: errMsg,
+						},
+					}, nil
+				}
+				for k, v := range envs {
+					deploy.Spec.Template.Spec.Containers[0].Env = append(deploy.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: k, Value: v})
+				}
+			} else {
+				log.Warnf("[mutation] /injection-deploy: the deployment %s/%s does not have the %s or %s label",
+					deploy.Name, deploy.Namespace, config.ServiceSpaceLabel, config.ApplicationLabel)
+			}
+		}
 		return &admissionv1.AdmissionResponse{
 			UID:     request.UID,
 			Allowed: true,
@@ -92,7 +118,7 @@ func injectionDeploy(request *admissionv1.AdmissionRequest) (*admissionv1.Admiss
 	}
 }
 
-func createConfigMap(deploy *appsv1.Deployment) error {
+func createOrUpdateConfigMap(deploy *appsv1.Deployment) error {
 	configMapData := config.DefaultInjectorConfigMap
 	if version, ok := deploy.Spec.Template.Labels[config.AgentVersionLabel]; ok {
 		if agentVersion, ok := config.InjectorAgentVersion[version]; ok {

pkg/mutation/mutation_deploy.go

@@ -0,0 +1,50 @@
+package resource
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/jd-opensource/joylive-injector/pkg/config"
+	"io"
+	"net/http"
+)
+
+type Response struct {
+	Error error       `json:"error,omitempty"`
+	Data  interface{} `json:"data,omitempty"`
+}
+
+type ApplicationEnvResponse struct {
+	Response
+	Data map[string]string `json:"data"`
+}
+
+func GetApplicationEnvironments(namespace, application string) (map[string]string, error) {
+	url := fmt.Sprintf(
+		"%s/v1/getApplicationEnvironment?application=%s&namespace=%s",
+		config.ControlPlaneUrl, application, namespace,
+	)
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("request failed, status: %s", resp.Status)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	var response ApplicationEnvResponse
+	if err := json.Unmarshal(body, &response); err != nil {
+		return nil, err
+	}
+
+	if response.Error != nil {
+		return nil, response.Error
+	}
+	return response.Data, nil
+}

pkg/resource/control.go

@@ -1,4 +1,4 @@
-package config
+package watcher
 
 import (
 	"context"
@@ -7,6 +7,7 @@ import (
 	clientset "github.com/jd-opensource/joylive-injector/client-go/clientset/versioned"
 	"github.com/jd-opensource/joylive-injector/client-go/informers/externalversions"
 	listerv1 "github.com/jd-opensource/joylive-injector/client-go/listers/injector/v1"
+	"github.com/jd-opensource/joylive-injector/pkg/config"
 	"github.com/jd-opensource/joylive-injector/pkg/log"
 	"go.uber.org/zap"
 	"gopkg.in/yaml.v3"
@@ -120,7 +121,7 @@ func (w *AgentVersionWatcher) cacheAgentVersion(agentVersion *v1.AgentVersion) e
 	avSpec := string(avSpecBytes)
 	log.Info("Received AgentVersion update event, start updating local configuration.", zap.String("agentVersion", agentVersion.Name),
 		zap.String("agentVersionSpec", avSpec))
-	InjectorAgentVersion[agentVersion.Spec.Version] = agentVersion.Spec
+	config.InjectorAgentVersion[agentVersion.Spec.Version] = agentVersion.Spec
 	return nil
 }
 

pkg/config/av_watcher.go pkg/watcher/av_watcher.gopkg/config/av_watcher.go renamed to pkg/watcher/av_watcher.go

@@ -1,8 +1,9 @@
-package config
+package watcher
 
 import (
 	"context"
 	"fmt"
+	"github.com/jd-opensource/joylive-injector/pkg/config"
 	"github.com/jd-opensource/joylive-injector/pkg/log"
 	"go.uber.org/zap"
 	"gopkg.in/yaml.v3"
@@ -30,7 +31,7 @@ func NewConfigMapWatcher(kubeClient kubernetes.Interface) *ConfigMapWatcher {
 	factory := informers.NewSharedInformerFactoryWithOptions(
 		kubeClient,
 		time.Second*10,
-		informers.WithNamespace(GetNamespace()),
+		informers.WithNamespace(config.GetNamespace()),
 		informers.WithTweakListOptions(func(options *metav1.ListOptions) {
 			options.LabelSelector = labels.SelectorFromSet(map[string]string{
 				"app": "joylive-injector",
@@ -123,18 +124,18 @@ func (w *ConfigMapWatcher) cacheConfigMap(configMap *v1.ConfigMap) error {
 	cmDataString := string(cmDataBytes)
 	log.Info("Received ConfigMap update event, start updating local configuration.", zap.String("cm", configMap.Name),
 		zap.String("data", cmDataString))
-	if configMap.Name == os.Getenv(ConfigMapEnvName) {
-		DefaultInjectorConfigMap = configMap.Data
-		if data, ok := configMap.Data[InjectorConfigName]; ok {
-			c, err := GetAgentInjectConfig(data)
+	if configMap.Name == os.Getenv(config.ConfigMapEnvName) {
+		config.DefaultInjectorConfigMap = configMap.Data
+		if data, ok := configMap.Data[config.InjectorConfigName]; ok {
+			c, err := config.GetAgentInjectConfig(data)
 			if err != nil {
 				return err
 			}
-			DefaultInjectorConfig = c
-			delete(DefaultInjectorConfigMap, InjectorConfigName)
+			config.DefaultInjectorConfig = c
+			delete(config.DefaultInjectorConfigMap, config.InjectorConfigName)
 		}
 	} else {
-		InjectorConfigMaps[configMap.Name] = configMap.Data
+		config.InjectorConfigMaps[configMap.Name] = configMap.Data
 	}
 	return nil
 }