Skip to content

Commit b0da1ac

Browse files
landylee007landylee007
authored
Merge pull request #4285 from PetrusZ/ft/improve_build_storage
support multi storage in build and freestyle job
2 parents b1f528a + afd9e4b commit b0da1ac

File tree

8 files changed

+114
-62
lines changed

8 files changed

+114
-62
lines changed

‎pkg/microservice/aslan/core/common/repository/models/build.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -99,8 +99,8 @@ type PreBuild struct {
9999
ClusterSource string `bson:"cluster_source" json:"cluster_source"`
100100
StrategyID string `bson:"strategy_id" json:"strategy_id"`
101101
// UseHostDockerDaemon determines is dockerDaemon on host node is used in pod
102-
UseHostDockerDaemon bool `bson:"use_host_docker_daemon" json:"use_host_docker_daemon"`
103-
TemporaryStorage *TemporaryStorage `bson:"temporary_storage" json:"temporary_storage"`
102+
UseHostDockerDaemon bool `bson:"use_host_docker_daemon" json:"use_host_docker_daemon"`
103+
Storages *Storages `bson:"storages" json:"storages"`
104104

105105
CustomAnnotations []*util.KeyValue `bson:"custom_annotations" json:"custom_annotations" yaml:"custom_annotations"`
106106
CustomLabels []*util.KeyValue `bson:"custom_labels" json:"custom_labels" yaml:"custom_labels"`
@@ -109,9 +109,9 @@ type PreBuild struct {
109109
Namespace string `bson:"namespace" json:"namespace"`
110110
}
111111

112-
type TemporaryStorage struct {
113-
Enabled bool `bson:"enabled" json:"enabled"`
114-
*types.NFSProperties `bson:",inline" json:",inline"`
112+
type Storages struct {
113+
Enabled bool `bson:"enabled" json:"enabled" yaml:"enabled"`
114+
StoragesProperties []*types.NFSProperties `bson:"storages_properties" json:"storages_properties" yaml:"storages_properties"`
115115
}
116116

117117
type PreDeploy struct {

‎pkg/microservice/aslan/core/common/repository/models/workflow_v4.go‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1417,6 +1417,7 @@ type JobAdvancedSettings struct {
14171417
CustomLabels []*util.KeyValue `bson:"custom_labels" json:"custom_labels" yaml:"custom_labels"`
14181418
// 共享存储配置
14191419
ShareStorageInfo *ShareStorageInfo `bson:"share_storage_info" json:"share_storage_info" yaml:"share_storage_info"`
1420+
Storages *Storages `bson:"storages" json:"storages" yaml:"storages"`
14201421
}
14211422

14221423
type JobProperties struct {
@@ -1434,19 +1435,19 @@ type JobProperties struct {
14341435
Namespace string `bson:"namespace" json:"namespace" yaml:"namespace"`
14351436
Envs KeyValList `bson:"envs" json:"envs" yaml:"envs"`
14361437
// log user-defined variables, shows in workflow task detail.
1437-
CustomEnvs []*KeyVal `bson:"custom_envs" json:"custom_envs" yaml:"custom_envs,omitempty"`
1438-
Params []*Param `bson:"params" json:"params" yaml:"params"`
1439-
LogFileName string `bson:"log_file_name" json:"log_file_name" yaml:"log_file_name"`
1440-
DockerHost string `bson:"-" json:"docker_host,omitempty" yaml:"docker_host,omitempty"`
1441-
Registries []*RegistryNamespace `bson:"registries" json:"registries" yaml:"registries"`
1442-
Cache types.Cache `bson:"cache" json:"cache" yaml:"cache"`
1443-
CacheEnable bool `bson:"cache_enable" json:"cache_enable" yaml:"cache_enable"`
1444-
CacheDirType types.CacheDirType `bson:"cache_dir_type" json:"cache_dir_type" yaml:"cache_dir_type"`
1445-
CacheUserDir string `bson:"cache_user_dir" json:"cache_user_dir" yaml:"cache_user_dir"`
1446-
ShareStorageDetails []*StorageDetail `bson:"share_storage_details" json:"share_storage_details" yaml:"-"`
1447-
EnablePrivileged bool `bson:"enable_privileged,omitempty" json:"enable_privileged,omitempty" yaml:"enable_privileged,omitempty"`
1448-
UseHostDockerDaemon bool `bson:"use_host_docker_daemon,omitempty" json:"use_host_docker_daemon,omitempty" yaml:"use_host_docker_daemon"`
1449-
TemporaryStorage *types.NFSProperties `bson:"temporary_storage" json:"temporary_storage" yaml:"temporary_storage"`
1438+
CustomEnvs []*KeyVal `bson:"custom_envs" json:"custom_envs" yaml:"custom_envs,omitempty"`
1439+
Params []*Param `bson:"params" json:"params" yaml:"params"`
1440+
LogFileName string `bson:"log_file_name" json:"log_file_name" yaml:"log_file_name"`
1441+
DockerHost string `bson:"-" json:"docker_host,omitempty" yaml:"docker_host,omitempty"`
1442+
Registries []*RegistryNamespace `bson:"registries" json:"registries" yaml:"registries"`
1443+
Cache types.Cache `bson:"cache" json:"cache" yaml:"cache"`
1444+
CacheEnable bool `bson:"cache_enable" json:"cache_enable" yaml:"cache_enable"`
1445+
CacheDirType types.CacheDirType `bson:"cache_dir_type" json:"cache_dir_type" yaml:"cache_dir_type"`
1446+
CacheUserDir string `bson:"cache_user_dir" json:"cache_user_dir" yaml:"cache_user_dir"`
1447+
ShareStorageDetails []*StorageDetail `bson:"share_storage_details" json:"share_storage_details" yaml:"-"`
1448+
EnablePrivileged bool `bson:"enable_privileged,omitempty" json:"enable_privileged,omitempty" yaml:"enable_privileged,omitempty"`
1449+
UseHostDockerDaemon bool `bson:"use_host_docker_daemon,omitempty" json:"use_host_docker_daemon,omitempty" yaml:"use_host_docker_daemon"`
1450+
Storages []*types.NFSProperties `bson:"storages" json:"storages" yaml:"storages"`
14501451
// for VM deploy to get service name to save
14511452
ServiceName string `bson:"service_name" json:"service_name" yaml:"service_name"`
14521453

‎pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_freestyle.go‎

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"time"
2525

2626
"github.com/koderover/zadig/v2/pkg/tool/clientmanager"
27+
"github.com/koderover/zadig/v2/pkg/types"
2728
"github.com/pkg/errors"
2829
"go.uber.org/zap"
2930
"gopkg.in/yaml.v2"
@@ -200,16 +201,20 @@ func (c *FreestyleJobCtl) run(ctx context.Context) error {
200201

201202
c.logger.Infof("succeed to create cm for job %s", c.job.K8sJobName)
202203

203-
if c.jobTaskSpec.Properties.TemporaryStorage != nil {
204-
err = service.CreateDynamicPVC(c.jobTaskSpec.Properties.ClusterID, getTemporaryStoragePVCName(c.job.K8sJobName), c.jobTaskSpec.Properties.TemporaryStorage, c.logger)
205-
if err != nil {
206-
msg := fmt.Sprintf("create dynamic PVC error: %v", err)
207-
logError(c.job, msg, c.logger)
208-
return errors.New(msg)
204+
if len(c.jobTaskSpec.Properties.Storages) > 0 {
205+
for i, storage := range c.jobTaskSpec.Properties.Storages {
206+
if storage.ProvisionType == types.DynamicProvision {
207+
err = service.CreateDynamicPVC(c.jobTaskSpec.Properties.ClusterID, getStoragePVCName(c.job.K8sJobName, i), storage, c.logger)
208+
if err != nil {
209+
msg := fmt.Sprintf("create dynamic PVC error: %v", err)
210+
logError(c.job, msg, c.logger)
211+
return errors.New(msg)
209212

210-
}
213+
}
211214

212-
c.logger.Infof("succeed to create dynamic PVC for job %s", c.job.K8sJobName)
215+
c.logger.Infof("succeed to create dynamic PVC for job %s", c.job.K8sJobName)
216+
}
217+
}
213218
}
214219

215220
jobImage := getBaseImage(c.jobTaskSpec.Properties.BuildOS, c.jobTaskSpec.Properties.ImageFrom)
@@ -357,9 +362,13 @@ func (c *FreestyleJobCtl) complete(ctx context.Context) {
357362
// 清理用户取消和超时的任务
358363
defer func() {
359364
go func() {
360-
if c.jobTaskSpec.Properties.TemporaryStorage != nil {
361-
if err := ensureDeletePVC(c.job.K8sJobName, c.jobTaskSpec.Properties.Namespace, c.jobTaskSpec.Properties.TemporaryStorage, c.kubeclient); err != nil {
362-
c.logger.Error(err)
365+
if len(c.jobTaskSpec.Properties.Storages) > 0 {
366+
for _, storage := range c.jobTaskSpec.Properties.Storages {
367+
if storage.IsTemporary {
368+
if err := ensureDeletePVC(storage.PVC, c.jobTaskSpec.Properties.Namespace, storage, c.kubeclient); err != nil {
369+
c.logger.Error(err)
370+
}
371+
}
363372
}
364373
}
365374
if err := ensureDeleteJob(c.jobTaskSpec.Properties.Namespace, jobLabel, c.kubeclient); err != nil {

‎pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/kubernetes.go‎

Lines changed: 22 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -112,12 +112,11 @@ type JobLabel struct {
112112
JobType string
113113
}
114114

115-
func getTemporaryStoragePVCName(k8sJobName string) string {
116-
return fmt.Sprintf("%s-temporary", k8sJobName)
115+
func getStoragePVCName(k8sJobName string, index int) string {
116+
return fmt.Sprintf("%s-%d", k8sJobName, index)
117117
}
118118

119-
func ensureDeletePVC(jobName, namespace string, storage *types.NFSProperties, kubeClient crClient.Client) error {
120-
pvcName := service.GetPVCName(getTemporaryStoragePVCName(jobName), storage)
119+
func ensureDeletePVC(pvcName, namespace string, storage *types.NFSProperties, kubeClient crClient.Client) error {
121120
return kubeClient.Delete(context.TODO(), &corev1.PersistentVolumeClaim{
122121
ObjectMeta: metav1.ObjectMeta{
123122
Name: pvcName,
@@ -481,7 +480,7 @@ EOF`,
481480
},
482481
}
483482

484-
setJobTemporaryStorages(job, workflowCtx, jobTaskSpec.Properties.TemporaryStorage, targetCluster)
483+
setJobStorages(job, workflowCtx, jobTaskSpec.Properties.Storages, targetCluster)
485484
setJobShareStorages(job, workflowCtx, jobTaskSpec.Properties.ShareStorageDetails, targetCluster)
486485

487486
if jobTaskSpec.Properties.CacheEnable && jobTaskSpec.Properties.Cache.MediumType == commontypes.NFSMedium {
@@ -569,29 +568,32 @@ func BuildCleanJob(jobName, clusterID, workflowName string, taskID int64) (*batc
569568
return job, nil
570569
}
571570

572-
func setJobTemporaryStorages(job *batchv1.Job, workflowCtx *commonmodels.WorkflowTaskCtx, temporaryStorage *types.NFSProperties, cluster *commonmodels.K8SCluster) {
573-
if temporaryStorage == nil {
571+
func setJobStorages(job *batchv1.Job, workflowCtx *commonmodels.WorkflowTaskCtx, storages []*types.NFSProperties, cluster *commonmodels.K8SCluster) {
572+
if len(storages) <= 0 {
574573
return
575574
}
576575

577576
// save cluster id so we can clean up share storage later
578577
workflowCtx.ClusterIDAdd(cluster.ID.Hex())
579578

580-
volumeName := "temporary-storage"
581-
job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{
582-
Name: volumeName,
583-
VolumeSource: corev1.VolumeSource{
584-
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
585-
ClaimName: temporaryStorage.PVC,
579+
for _, storage := range storages {
580+
volumeName := fmt.Sprintf("storage-%s", storage.PVC)
581+
job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{
582+
Name: volumeName,
583+
VolumeSource: corev1.VolumeSource{
584+
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
585+
ClaimName: storage.PVC,
586+
},
586587
},
587-
},
588-
})
588+
})
589+
590+
job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
591+
Name: volumeName,
592+
MountPath: storage.MountPath,
593+
SubPath: storage.Subpath,
594+
})
595+
}
589596

590-
job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
591-
Name: volumeName,
592-
MountPath: "/workspace",
593-
SubPath: fmt.Sprintf("%s/%s", job.Name, "temporary-storage"),
594-
})
595597
}
596598

597599
func setJobShareStorages(job *batchv1.Job, workflowCtx *commonmodels.WorkflowTaskCtx, storageDetails []*commonmodels.StorageDetail, cluster *commonmodels.K8SCluster) {

‎pkg/microservice/aslan/core/multicluster/service/clusters.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1402,7 +1402,7 @@ func CreateDynamicPVC(clusterID, prefix string, nfsProperties *types.NFSProperti
14021402

14031403
accessMode := []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}
14041404
if nfsProperties.AccessMode != "" {
1405-
accessMode = []corev1.PersistentVolumeAccessMode{corev1.PersistentVolumeAccessMode(nfsProperties.AccessMode)}
1405+
accessMode = []corev1.PersistentVolumeAccessMode{nfsProperties.AccessMode}
14061406
}
14071407

14081408
pvc = &corev1.PersistentVolumeClaim{

‎pkg/microservice/aslan/core/workflow/service/workflow/controller/job/job_build.go‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import (
2323
"strings"
2424

2525
"go.uber.org/zap"
26-
corev1 "k8s.io/api/core/v1"
2726
"k8s.io/apimachinery/pkg/util/sets"
2827

2928
configbase "github.com/koderover/zadig/v2/pkg/config"
@@ -41,6 +40,7 @@ import (
4140
"github.com/koderover/zadig/v2/pkg/types"
4241
"github.com/koderover/zadig/v2/pkg/types/job"
4342
"github.com/koderover/zadig/v2/pkg/types/step"
43+
pkgutil "github.com/koderover/zadig/v2/pkg/util"
4444
)
4545

4646
// TODO: Change note: ServiceAndBuilds field use to be the option field for the configuration, it has been
@@ -400,11 +400,6 @@ func (j BuildJobController) ToTask(taskID int64) ([]*commonmodels.JobTask, error
400400
EnablePrivileged: buildInfo.EnablePrivilegedMode,
401401
}
402402

403-
if buildInfo.PreBuild != nil && buildInfo.PreBuild.TemporaryStorage != nil && buildInfo.PreBuild.TemporaryStorage.Enabled {
404-
jobTaskSpec.Properties.TemporaryStorage = buildInfo.PreBuild.TemporaryStorage.NFSProperties
405-
jobTaskSpec.Properties.TemporaryStorage.AccessMode = string(corev1.ReadWriteOnce)
406-
}
407-
408403
paramEnvs := generateKeyValsFromWorkflowParam(j.workflow.Params)
409404
envs := mergeKeyVals(jobTaskSpec.Properties.CustomEnvs, paramEnvs)
410405
renderedEnv, err := replaceServiceAndModules(envs, build.ServiceName, build.ServiceModule)
@@ -415,6 +410,25 @@ func (j BuildJobController) ToTask(taskID int64) ([]*commonmodels.JobTask, error
415410
jobTaskSpec.Properties.Envs = append(renderedEnv, getBuildJobVariables(build, taskID, j.workflow.Project, j.workflow.Name, j.workflow.DisplayName, image, pkgFile, jobTask.Infrastructure, registry, logger)...)
416411
jobTaskSpec.Properties.UseHostDockerDaemon = buildInfo.PreBuild.UseHostDockerDaemon
417412

413+
if buildInfo.PreBuild != nil && buildInfo.PreBuild.Storages != nil && buildInfo.PreBuild.Storages.Enabled {
414+
if len(buildInfo.PreBuild.Storages.StoragesProperties) > 0 {
415+
newStorages := make([]*types.NFSProperties, 0)
416+
for _, storage := range buildInfo.PreBuild.Storages.StoragesProperties {
417+
newStorage := &types.NFSProperties{}
418+
err = pkgutil.DeepCopy(newStorage, storage)
419+
if err != nil {
420+
return nil, fmt.Errorf("failed to deep copy storage: %v", err)
421+
}
422+
423+
newStorage.MountPath = commonutil.RenderEnv(storage.MountPath, jobTaskSpec.Properties.Envs)
424+
newStorage.Subpath = commonutil.RenderEnv(storage.Subpath, jobTaskSpec.Properties.Envs)
425+
newStorages = append(newStorages, newStorage)
426+
}
427+
428+
jobTaskSpec.Properties.Storages = newStorages
429+
}
430+
}
431+
418432
cacheS3 := &commonmodels.S3Storage{}
419433
if jobTask.Infrastructure == setting.JobVMInfrastructure {
420434
jobTaskSpec.Properties.CacheEnable = buildInfo.CacheEnable

‎pkg/microservice/aslan/core/workflow/service/workflow/controller/job/job_freestyle.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import (
3232
"github.com/koderover/zadig/v2/pkg/tool/log"
3333
"github.com/koderover/zadig/v2/pkg/types"
3434
steptypes "github.com/koderover/zadig/v2/pkg/types/step"
35+
pkgutil "github.com/koderover/zadig/v2/pkg/util"
3536
util2 "github.com/koderover/zadig/v2/pkg/util"
3637
)
3738

@@ -571,6 +572,25 @@ func (j FreestyleJobController) generateSubTask(taskID int64, jobSubTaskID int,
571572
taskRunProperties.Envs = renderedEnvs
572573
}
573574

575+
if j.jobSpec.AdvancedSetting.Storages != nil && j.jobSpec.AdvancedSetting.Storages.Enabled {
576+
if len(j.jobSpec.AdvancedSetting.Storages.StoragesProperties) > 0 {
577+
newStorages := make([]*types.NFSProperties, 0)
578+
for _, storage := range j.jobSpec.AdvancedSetting.Storages.StoragesProperties {
579+
newStorage := &types.NFSProperties{}
580+
err = pkgutil.DeepCopy(newStorage, storage)
581+
if err != nil {
582+
return nil, fmt.Errorf("failed to deep copy storage: %v", err)
583+
}
584+
585+
newStorage.MountPath = commonutil.RenderEnv(storage.MountPath, taskRunProperties.Envs)
586+
newStorage.Subpath = commonutil.RenderEnv(storage.Subpath, taskRunProperties.Envs)
587+
newStorages = append(newStorages, newStorage)
588+
}
589+
590+
taskRunProperties.Storages = newStorages
591+
}
592+
}
593+
574594
repos := j.jobSpec.Repos
575595
if service != nil {
576596
repos = service.Repos

‎pkg/types/cache.go‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,10 @@ limitations under the License.
1616

1717
package types
1818

19+
import (
20+
corev1 "k8s.io/api/core/v1"
21+
)
22+
1923
type MediumType string
2024

2125
const (
@@ -40,12 +44,14 @@ type ObjectProperties struct {
4044
}
4145

4246
type NFSProperties struct {
43-
ProvisionType ProvisionType `json:"provision_type" bson:"provision_type" yaml:"provision_type"`
44-
StorageClass string `json:"storage_class" bson:"storage_class" yaml:"storage_class"`
45-
StorageSizeInGiB int64 `json:"storage_size_in_gib" bson:"storage_size_in_gib" yaml:"storage_size_in_gib"`
46-
PVC string `json:"pvc" bson:"pvc" yaml:"pvc"`
47-
Subpath string `json:"subpath" bson:"subpath" yaml:"subpath"`
48-
AccessMode string `json:"access_mode" bson:"access_mode" yaml:"access_mode"`
47+
ProvisionType ProvisionType `json:"provision_type" bson:"provision_type" yaml:"provision_type"`
48+
StorageClass string `json:"storage_class" bson:"storage_class" yaml:"storage_class"`
49+
StorageSizeInGiB int64 `json:"storage_size_in_gib" bson:"storage_size_in_gib" yaml:"storage_size_in_gib"`
50+
PVC string `json:"pvc" bson:"pvc" yaml:"pvc"`
51+
AccessMode corev1.PersistentVolumeAccessMode `json:"access_mode" bson:"access_mode" yaml:"access_mode"`
52+
Subpath string `json:"sub_path" bson:"sub_path" yaml:"sub_path"`
53+
MountPath string `json:"mount_path" bson:"mount_path" yaml:"mount_path"`
54+
IsTemporary bool `json:"is_temporary" bson:"is_temporary" yaml:"is_temporary"`
4955
}
5056

5157
type Cache struct {

0 commit comments

Comments
 (0)