Support configuring the operatiom image in the vineyardd CR and update relevant doc (#1165)

dashanji · web-flow · commit 64db68f11bf7 · 2023-01-31T20:11:16.000+08:00
* Support configuring the operation image in the vineyardd CR.
* Update the relevant doc.
* Generate the corresponding vineyard operator chart.

Signed-off-by: Ye Cao &lt;caoye.cao@alibaba-inc.com&gt;
diff --git a/charts/vineyard-operator/templates/vineyardd-crd.yaml b/charts/vineyard-operator/templates/vineyardd-crd.yaml
@@ -70,6 +70,30 @@ spec:
                     default: IfNotPresent
                     type: string
                 type: object
+              pluginConfig:
+                default:
+                  backupImage: ghcr.io/v6d-io/v6d/backup-job
+                  daskRepartitionImage: ghcr.io/v6d-io/v6d/dask-repartition
+                  distributedAssemblyImage: ghcr.io/v6d-io/v6d/distributed-assembly
+                  localAssemblyImage: ghcr.io/v6d-io/v6d/local-assembly
+                  recoverImage: ghcr.io/v6d-io/v6d/recover-job
+                properties:
+                  backupImage:
+                    default: ghcr.io/v6d-io/v6d/backup-job
+                    type: string
+                  daskRepartitionImage:
+                    default: ghcr.io/v6d-io/v6d/dask-repartition
+                    type: string
+                  distributedAssemblyImage:
+                    default: ghcr.io/v6d-io/v6d/distributed-assembly
+                    type: string
+                  localAssemblyImage:
+                    default: ghcr.io/v6d-io/v6d/local-assembly
+                    type: string
+                  recoverImage:
+                    default: ghcr.io/v6d-io/v6d/recover-job
+                    type: string
+                type: object
               replicas:
                 default: 3
                 type: integer
diff --git a/docs/notes/vineyard-operator.rst b/docs/notes/vineyard-operator.rst
@@ -308,11 +308,41 @@ The detailed configuration entries for creating a vineyard cluster are listed as
      - The environment of vineyardd.
      - nil
 
-   * - | metricConfig.
-       | enable
-     - bool
-     - Enable the metrics in vineyardd.
-     - false
+   * - | vineyardConfig.
+       | env
+     - []corev1.EnvVar
+     - The environment of vineyardd.
+     - nil
+
+   * - | pluginConfig.
+       | backupImage
+     - string
+     - The image of backup operation
+     - "ghcr.io/v6d-io/v6d/backup-job"
+
+   * - | pluginConfig.
+       | recoverImage
+     - string
+     - The image of recover operation
+     - "ghcr.io/v6d-io/v6d/recover-job"
+
+   * - | pluginConfig.
+       | daskRepartitionImage
+     - string
+     - The image of dask repartition operation
+     - "ghcr.io/v6d-io/v6d/dask-repartition"
+
+   * - | pluginConfig.
+       | localAssemblyImage
+     - string
+     - The image of local assembly operation
+     - "ghcr.io/v6d-io/v6d/local-assembly"
+
+   * - | pluginConfig.
+       | distributedAssemblyImage
+     - string
+     - The image of distributed assembly operation
+     - "ghcr.io/v6d-io/v6d/distributed-assembly"
 
    * - | metricConfig.
        | image
diff --git a/k8s/apis/k8s/v1alpha1/vineyardd_types.go b/k8s/apis/k8s/v1alpha1/vineyardd_types.go
@@ -167,6 +167,35 @@ type VineyardContainerConfig struct {
 	Env []corev1.EnvVar `json:"env,omitempty"`
 }
 
+// PluginImageConfig holds all image configuration about pluggable drivers(backup, recover,
+// local assembly, distributed assembly, repartition)
+type PluginImageConfig struct {
+	// the image of backup operation
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default:="ghcr.io/v6d-io/v6d/backup-job"
+	BackupImage string `json:"backupImage,omitempty"`
+
+	// the image of recover operation
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default:="ghcr.io/v6d-io/v6d/recover-job"
+	RecoverImage string `json:"recoverImage,omitempty"`
+
+	// the image of dask repartition operation
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default:="ghcr.io/v6d-io/v6d/dask-repartition"
+	DaskRepartitionImage string `json:"daskRepartitionImage,omitempty"`
+
+	// the image of local assembly operation
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default:="ghcr.io/v6d-io/v6d/local-assembly"
+	LocalAssemblyImage string `json:"localAssemblyImage,omitempty"`
+
+	// the image of distributed assembly operation
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default:="ghcr.io/v6d-io/v6d/distributed-assembly"
+	DistributedAssemblyImage string `json:"distributedAssemblyImage,omitempty"`
+}
+
 // VineyarddSpec holds all configuration about vineyardd
 type VineyarddSpec struct {
 	// the replicas of vineyardd
@@ -190,6 +219,12 @@ type VineyarddSpec struct {
 	// +kubebuilder:default:={image: "vineyardcloudnative/vineyardd:latest", imagePullPolicy: "IfNotPresent", syncCRDs: true, socket: "/var/run/vineyard-kubernetes/{{.Namespace}}/{{.Name}}", size: "256Mi", streamThreshold: 80, etcdEndpoint: "http://etcd-for-vineyard:2379", etcdPrefix: "/vineyard"}
 	VineyardConfig VineyardContainerConfig `json:"vineyardConfig,omitempty"`
 
+	// operation container configuration
+	// +kubebuilder:validation:Optional
+	//nolint: lll
+	// +kubebuilder:default={backupImage: "ghcr.io/v6d-io/v6d/backup-job", recoverImage: "ghcr.io/v6d-io/v6d/recover-job", daskRepartitionImage: "ghcr.io/v6d-io/v6d/dask-repartition", localAssemblyImage: "ghcr.io/v6d-io/v6d/local-assembly", distributedAssemblyImage: "ghcr.io/v6d-io/v6d/distributed-assembly"}
+	PluginConfig PluginImageConfig `json:"pluginConfig,omitempty"`
+
 	// metric container configuration
 	// +kubebuilder:validation:Optional
 	// +kubebuilder:default:={enable: false, image: "vineyardcloudnative/vineyard-grok-exporter:latest", imagePullPolicy: "IfNotPresent"}
diff --git a/k8s/config/crd/bases/k8s.v6d.io_vineyardds.yaml b/k8s/config/crd/bases/k8s.v6d.io_vineyardds.yaml
@@ -58,6 +58,30 @@ spec:
                     default: IfNotPresent
                     type: string
                 type: object
+              pluginConfig:
+                default:
+                  backupImage: ghcr.io/v6d-io/v6d/backup-job
+                  daskRepartitionImage: ghcr.io/v6d-io/v6d/dask-repartition
+                  distributedAssemblyImage: ghcr.io/v6d-io/v6d/distributed-assembly
+                  localAssemblyImage: ghcr.io/v6d-io/v6d/local-assembly
+                  recoverImage: ghcr.io/v6d-io/v6d/recover-job
+                properties:
+                  backupImage:
+                    default: ghcr.io/v6d-io/v6d/backup-job
+                    type: string
+                  daskRepartitionImage:
+                    default: ghcr.io/v6d-io/v6d/dask-repartition
+                    type: string
+                  distributedAssemblyImage:
+                    default: ghcr.io/v6d-io/v6d/distributed-assembly
+                    type: string
+                  localAssemblyImage:
+                    default: ghcr.io/v6d-io/v6d/local-assembly
+                    type: string
+                  recoverImage:
+                    default: ghcr.io/v6d-io/v6d/recover-job
+                    type: string
+                type: object
               replicas:
                 default: 3
                 type: integer
diff --git a/k8s/config/manager/kustomization.yaml b/k8s/config/manager/kustomization.yaml
@@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: controller
-  newName: localhost:5001/vineyard-operator
+  newName: vineyardcloudnative/vineyard-operator
   newTag: latest
diff --git a/k8s/controllers/k8s/operation_controller.go b/k8s/controllers/k8s/operation_controller.go
@@ -55,6 +55,7 @@ type OperationReconciler struct {
 // +kubebuilder:rbac:groups="",resources=pods/status,verbs=get;list;watch;create;update;patch
 // +kubebuilder:rbac:groups="",resources=pods/log,verbs=get
 // +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=k8s.v6d.io,resources=vineyardds,verbs=get;list;watch;create;update;patch;delete
 
 // Reconcile reconciles the operation
 func (r *OperationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
@@ -64,7 +65,7 @@ func (r *OperationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	if err := r.Client.Get(ctx, req.NamespacedName, &op); err != nil {
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
-	logger.Info("Reconciling Operation", "vineyardd", op)
+	logger.Info("Reconciling Operation", "operation", op)
 
 	app := kubernetes.Application{
 		Client:   r.Client,
diff --git a/k8s/pkg/operation/assembly.go b/k8s/pkg/operation/assembly.go
@@ -52,25 +52,27 @@ type AssemblyOperation struct {
 
 // LocalAssemblyConfig is the config for the local assembly job
 type LocalAssemblyConfig struct {
-	Name             string
-	Namespace        string
-	StreamID         string
-	JobName          string
-	NodeName         string
-	TimeoutSeconds   int64
-	VineyardSockPath string
+	Name               string
+	Namespace          string
+	StreamID           string
+	JobName            string
+	NodeName           string
+	TimeoutSeconds     int64
+	VineyardSockPath   string
+	LocalAssemblyImage string
 }
 
 // DistributedAssemblyConfig is the config for the distributed assembly job
 type DistributedAssemblyConfig struct {
-	Name                 string
-	Namespace            string
-	StreamID             string
-	JobName              string
-	GlobalObjectID       string
-	OldObjectToNewObject string
-	TimeoutSeconds       int64
-	VineyardSockPath     string
+	Name                     string
+	Namespace                string
+	StreamID                 string
+	JobName                  string
+	GlobalObjectID           string
+	OldObjectToNewObject     string
+	TimeoutSeconds           int64
+	VineyardSockPath         string
+	DistributedAssemblyImage string
 }
 
 // LocalAssemblyConfigTemplate is the template config for the assembly job
@@ -146,6 +148,18 @@ func (ao *AssemblyOperation) findNeedAssemblyPodByLocalObject(ctx context.Contex
 // buildLocalAssemblyJob build all configuration for the local assembly job
 func (ao *AssemblyOperation) buildLocalAssemblyJob(ctx context.Context, localObject *v1alpha1.LocalObject, pod *corev1.Pod, timeout int64) error {
 	podLabels := pod.Labels
+
+	vineyarddName := podLabels[labels.VineyarddName]
+	vineyarddNamespace := podLabels[labels.VineyarddNamespace]
+	// get vineyardd cluster info
+	vineyardd := &v1alpha1.Vineyardd{}
+	if err := ao.Client.Get(ctx, client.ObjectKey{
+		Name:      vineyarddName,
+		Namespace: vineyarddNamespace,
+	}, vineyardd); err != nil {
+		return fmt.Errorf("failed to get the vineyardd: %v", err)
+	}
+
 	// When the pod which generated the stream is annotated, the assembly job will be created in the same pod
 	if _, ok := podLabels[NeedInjectedAssemblyKey]; ok {
 		if strings.Contains(strings.ToLower(localObject.Spec.Typename), "stream") {
@@ -155,6 +169,7 @@ func (ao *AssemblyOperation) buildLocalAssemblyJob(ctx context.Context, localObj
 			LocalAssemblyConfigTemplate.NodeName = localObject.Spec.Hostname
 			LocalAssemblyConfigTemplate.JobName = podLabels[labels.VineyardJobName]
 			LocalAssemblyConfigTemplate.TimeoutSeconds = timeout
+			LocalAssemblyConfigTemplate.LocalAssemblyImage = vineyardd.Spec.PluginConfig.LocalAssemblyImage
 			if socket, err := ao.ResolveRequiredVineyarddSocket(
 				ctx,
 				podLabels[labels.VineyarddName],
@@ -286,6 +301,17 @@ func (ao *AssemblyOperation) buildDistributedAssemblyJob(
 		}
 	}
 
+	vineyarddName := podLabels[labels.VineyarddName]
+	vineyarddNamespace := podLabels[labels.VineyarddNamespace]
+	// get vineyardd cluster info
+	vineyardd := &v1alpha1.Vineyardd{}
+	if err := ao.Client.Get(ctx, client.ObjectKey{
+		Name:      vineyarddName,
+		Namespace: vineyarddNamespace,
+	}, vineyardd); err != nil {
+		return false, fmt.Errorf("failed to get the vineyardd: %v", err)
+	}
+
 	// build the distributed assembly job
 	if len(sigToID) == len(oldObjectToNewObject) {
 		str := `'{`
@@ -299,6 +325,7 @@ func (ao *AssemblyOperation) buildDistributedAssemblyJob(
 		DistributedAssemblyConfigTemplate.OldObjectToNewObject = str
 		DistributedAssemblyConfigTemplate.JobName = podLabels[labels.VineyardJobName]
 		DistributedAssemblyConfigTemplate.TimeoutSeconds = timeout
+		DistributedAssemblyConfigTemplate.DistributedAssemblyImage = vineyardd.Spec.PluginConfig.DistributedAssemblyImage
 		if socket, err := ao.ResolveRequiredVineyarddSocket(
 			ctx,
 			podLabels[labels.VineyarddName],
diff --git a/k8s/pkg/operation/repartition.go b/k8s/pkg/operation/repartition.go
@@ -55,15 +55,16 @@ type RepartitionOperation struct {
 
 // DaskRepartitionConfig is the config for the dask repartition job
 type DaskRepartitionConfig struct {
-	Name             string
-	Namespace        string
-	GlobalObjectID   string
-	Replicas         string
-	InstanceToWorker string
-	DaskScheduler    string
-	JobName          string
-	TimeoutSeconds   int64
-	VineyardSockPath string
+	Name                 string
+	Namespace            string
+	GlobalObjectID       string
+	Replicas             string
+	InstanceToWorker     string
+	DaskScheduler        string
+	JobName              string
+	TimeoutSeconds       int64
+	VineyardSockPath     string
+	DaskRepartitionImage string
 }
 
 // DaskRepartitionConfigTemplate is the template config for the dask repartition job
@@ -173,6 +174,17 @@ func (ro *RepartitionOperation) buildDaskRepartitionJob(ctx context.Context, glo
 		return fmt.Errorf("failed to get replicas from target jobs")
 	}
 
+	vineyarddName := pod.Labels[labels.VineyarddName]
+	vineyarddNamespace := pod.Labels[labels.VineyarddNamespace]
+	// get vineyardd cluster info
+	vineyardd := &v1alpha1.Vineyardd{}
+	if err := ro.Client.Get(ctx, client.ObjectKey{
+		Name:      vineyarddName,
+		Namespace: vineyarddNamespace,
+	}, vineyardd); err != nil {
+		return fmt.Errorf("failed to get the vineyardd: %v", err)
+	}
+
 	DaskRepartitionConfigTemplate.Replicas = "'" + replicas + "'"
 	DaskRepartitionConfigTemplate.Name = RepartitionPrefix + globalObject.Name
 	DaskRepartitionConfigTemplate.Namespace = pod.Namespace
@@ -181,6 +193,7 @@ func (ro *RepartitionOperation) buildDaskRepartitionJob(ctx context.Context, glo
 	DaskRepartitionConfigTemplate.JobName = pod.Labels[labels.VineyardJobName]
 	DaskRepartitionConfigTemplate.InstanceToWorker = instanceToWorker
 	DaskRepartitionConfigTemplate.TimeoutSeconds = o.Spec.TimeoutSeconds
+	DaskRepartitionConfigTemplate.DaskRepartitionImage = vineyardd.Spec.PluginConfig.DaskRepartitionImage
 	if socket, err := ro.ResolveRequiredVineyarddSocket(
 		ctx,
 		pod.Labels[labels.VineyarddName],
diff --git a/k8s/pkg/templates/operation/dask-repartition.yaml b/k8s/pkg/templates/operation/dask-repartition.yaml
@@ -25,7 +25,7 @@ spec:
     spec:
       containers:
       - name: dask-repartition-operation
-        image: ghcr.io/v6d-io/v6d/dask-repartition
+        image: {{ $config.DaskRepartitionImage }}
         imagePullPolicy: IfNotPresent
         env:
         - name: GLOBALOBJECT_ID
diff --git a/k8s/pkg/templates/operation/distributed-assembly-job.yaml b/k8s/pkg/templates/operation/distributed-assembly-job.yaml
@@ -25,7 +25,7 @@ spec:
     spec:
       containers:
       - name: assembly-operation
-        image: ghcr.io/v6d-io/v6d/distributed-assembly
+        image: {{ $config.DistributedAssemblyImage}}
         imagePullPolicy: IfNotPresent
         env:
         - name: GLOBALOBJECT_ID
diff --git a/k8s/pkg/templates/operation/local-assembly-job.yaml b/k8s/pkg/templates/operation/local-assembly-job.yaml
@@ -26,7 +26,7 @@ spec:
       nodeName: {{ $config.NodeName }}
       containers:
       - name: assembly-operation
-        image: ghcr.io/v6d-io/v6d/local-assembly
+        image: {{ $config.LocalAssemblyImage}}
         imagePullPolicy: IfNotPresent
         env:
         - name: STREAM_ID
diff --git a/k8s/test/e2e/Makefile b/k8s/test/e2e/Makefile
diff --git a/k8s/test/e2e/assembly/distributed-assembly-e2e.yaml b/k8s/test/e2e/assembly/distributed-assembly-e2e.yaml
diff --git a/k8s/test/e2e/vineyardd.yaml b/k8s/test/e2e/vineyardd.yaml
