feat: mount file type variable in ci-runner through cm

Author: Ash-exp

api/bean/ConfigMapAndSecret.go

@@ -57,14 +57,19 @@ type ConfigSecretMap struct {
 	SubPath        bool            `json:"subPath"`
 	ESOSubPath     []string        `json:"esoSubPath"`
 	FilePermission string          `json:"filePermission"`
+	ConfigSecretMapEnt
 }
 
-func (configSecret ConfigSecretMap) GetDataMap() (map[string]string, error) {
-	var datamap map[string]string
-	err := json.Unmarshal(configSecret.Data, &datamap)
-	return datamap, err
+func (configSecret *ConfigSecretMap) GetDataMap() (map[string]string, error) {
+	if len(configSecret.Data) == 0 {
+		return make(map[string]string), nil
+	}
+	var dataMap map[string]string
+	err := json.Unmarshal(configSecret.Data, &dataMap)
+	return dataMap, err
 }
-func (configSecretJson ConfigSecretJson) GetDereferencedSecrets() []ConfigSecretMap {
+
+func (configSecretJson *ConfigSecretJson) GetDereferencedSecrets() []ConfigSecretMap {
 	return sliceUtil.GetDeReferencedSlice(configSecretJson.Secrets)
 }
 
@@ -95,3 +100,14 @@ func GetTransformedDataForSecretRootJsonData(data string, mode util.SecretTransf
 	}
 	return string(marshal), nil
 }
+
+type ConfigType string
+
+func (c ConfigType) String() string {
+	return string(c)
+}
+
+const (
+	ConfigMap ConfigType = "cm"
+	Secret    ConfigType = "cs"
+)

api/bean/ConfigMapAndSecret_ent.go

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2024. Devtron Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package bean
+
+type ConfigSecretMapEnt struct {
+}
+
+func (configSecret *ConfigSecretMap) AddDataToKey(keyName string, data []byte) (*ConfigSecretMap, error) {
+	return configSecret, nil
+}
+
+func (configSecret *ConfigSecretMap) GetBinaryDataMap() map[string][]byte {
+	return nil
+}

pkg/deployment/trigger/devtronApps/preStageHandlerCode.go

@@ -23,7 +23,7 @@ import (
 	"github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
 	blob_storage "github.com/devtron-labs/common-lib/blob-storage"
 	commonBean "github.com/devtron-labs/common-lib/workflow"
-	bean2 "github.com/devtron-labs/devtron/api/bean"
+	apiBean "github.com/devtron-labs/devtron/api/bean"
 	gitSensorClient "github.com/devtron-labs/devtron/client/gitSensor"
 	constants2 "github.com/devtron-labs/devtron/internal/sql/constants"
 	"github.com/devtron-labs/devtron/internal/sql/repository"
@@ -62,7 +62,7 @@ import (
 )
 
 func (impl *HandlerServiceImpl) TriggerPreStage(request bean.TriggerRequest) (*bean6.ManifestPushTemplate, error) {
-	request.WorkflowType = bean2.CD_WORKFLOW_TYPE_PRE
+	request.WorkflowType = apiBean.CD_WORKFLOW_TYPE_PRE
 	// setting triggeredAt variable to have consistent data for various audit log places in db for deployment time
 	triggeredAt := time.Now()
 	triggeredBy := request.TriggeredBy
@@ -219,7 +219,7 @@ func (impl *HandlerServiceImpl) TriggerAutoCDOnPreStageSuccess(triggerContext be
 func (impl *HandlerServiceImpl) checkDeploymentTriggeredAlready(wfId int) bool {
 	deploymentTriggeredAlready := false
 	// TODO : need to check this logic for status check in case of multiple deployments requirement for same workflow
-	workflowRunner, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(context.Background(), wfId, bean2.CD_WORKFLOW_TYPE_DEPLOY)
+	workflowRunner, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(context.Background(), wfId, apiBean.CD_WORKFLOW_TYPE_DEPLOY)
 	if err != nil {
 		impl.logger.Errorw("error occurred while fetching workflow runner", "wfId", wfId, "err", err)
 		return deploymentTriggeredAlready
@@ -236,7 +236,7 @@ func (impl *HandlerServiceImpl) createStartingWfAndRunner(request bean.TriggerRe
 	//in case of pre stage manual trigger auth is already applied and for auto triggers there is no need for auth check here
 	cdWf := request.CdWf
 	var err error
-	if cdWf == nil && request.WorkflowType == bean2.CD_WORKFLOW_TYPE_PRE {
+	if cdWf == nil && request.WorkflowType == apiBean.CD_WORKFLOW_TYPE_PRE {
 		cdWf = &pipelineConfig.CdWorkflow{
 			CiArtifactId: artifact.Id,
 			PipelineId:   pipeline.Id,
@@ -279,9 +279,9 @@ func (impl *HandlerServiceImpl) getEnvAndNsIfRunStageInEnv(ctx context.Context,
 	var err error
 	namespace := impl.config.GetDefaultNamespace()
 	runStageInEnv := false
-	if workflowStage == bean2.CD_WORKFLOW_TYPE_PRE {
+	if workflowStage == apiBean.CD_WORKFLOW_TYPE_PRE {
 		runStageInEnv = pipeline.RunPreStageInEnv
-	} else if workflowStage == bean2.CD_WORKFLOW_TYPE_POST {
+	} else if workflowStage == apiBean.CD_WORKFLOW_TYPE_POST {
 		runStageInEnv = pipeline.RunPostStageInEnv
 	}
 	_, span := otel.Tracer("orchestrator").Start(ctx, "envRepository.FindById")
@@ -588,7 +588,7 @@ func (impl *HandlerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow
 	}
 	if pipelineStage != nil {
 		var variableSnapshot map[string]string
-		if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_PRE {
+		if runner.WorkflowType == apiBean.CD_WORKFLOW_TYPE_PRE {
 			// TODO: use const from pipeline.WorkflowService:95
 			request := pipelineConfigBean.NewBuildPrePostStepDataReq(cdPipeline.Id, "preCD", scope)
 			prePostAndRefPluginResponse, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(request)
@@ -599,7 +599,7 @@ func (impl *HandlerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow
 			preDeploySteps = prePostAndRefPluginResponse.PreStageSteps
 			refPluginsData = prePostAndRefPluginResponse.RefPluginData
 			variableSnapshot = prePostAndRefPluginResponse.VariableSnapshot
-		} else if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_POST {
+		} else if runner.WorkflowType == apiBean.CD_WORKFLOW_TYPE_POST {
 			// TODO: use const from pipeline.WorkflowService:96
 			request := pipelineConfigBean.NewBuildPrePostStepDataReq(cdPipeline.Id, "postCD", scope)
 			prePostAndRefPluginResponse, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(request)
@@ -630,9 +630,9 @@ func (impl *HandlerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow
 		}
 	} else {
 		//in this case no plugin script is not present for this cdPipeline hence going with attaching preStage or postStage config
-		if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_PRE {
+		if runner.WorkflowType == apiBean.CD_WORKFLOW_TYPE_PRE {
 			stageYaml = cdPipeline.PreStageConfig
-		} else if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_POST {
+		} else if runner.WorkflowType == apiBean.CD_WORKFLOW_TYPE_POST {
 			stageYaml = cdPipeline.PostStageConfig
 			deployStageWfr, deployStageTriggeredByUserEmail, pipelineReleaseCounter, err = impl.getDeployStageDetails(cdPipeline.Id)
 			if err != nil {
@@ -813,11 +813,11 @@ func (impl *HandlerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow
 	// For Pre-CD / Post-CD workflow, cache is not uploaded; hence no need to set cache bucket
 	cdWorkflowConfigCdCacheBucket := ""
 
-	if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_PRE {
+	if runner.WorkflowType == apiBean.CD_WORKFLOW_TYPE_PRE {
 		// populate input variables of steps with extra env variables
 		setExtraEnvVariableInDeployStep(preDeploySteps, runtimeParams.GetSystemVariables(), webhookAndCiData)
 		cdStageWorkflowRequest.PrePostDeploySteps = preDeploySteps
-	} else if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_POST {
+	} else if runner.WorkflowType == apiBean.CD_WORKFLOW_TYPE_POST {
 		// populate input variables of steps with extra env variables
 		setExtraEnvVariableInDeployStep(postDeploySteps, runtimeParams.GetSystemVariables(), webhookAndCiData)
 		cdStageWorkflowRequest.PrePostDeploySteps = postDeploySteps
@@ -1007,7 +1007,7 @@ func setExtraEnvVariableInDeployStep(deploySteps []*pipelineConfigBean.StepObjec
 func (impl *HandlerServiceImpl) getDeployStageDetails(pipelineId int) (pipelineConfig.CdWorkflowRunner, string, int, error) {
 	deployStageWfr := pipelineConfig.CdWorkflowRunner{}
 	//getting deployment pipeline latest wfr by pipelineId
-	deployStageWfr, err := impl.cdWorkflowRepository.FindLatestByPipelineIdAndRunnerType(pipelineId, bean2.CD_WORKFLOW_TYPE_DEPLOY)
+	deployStageWfr, err := impl.cdWorkflowRepository.FindLatestByPipelineIdAndRunnerType(pipelineId, apiBean.CD_WORKFLOW_TYPE_DEPLOY)
 	if err != nil {
 		impl.logger.Errorw("error in getting latest status of deploy type wfr by pipelineId", "err", err, "pipelineId", pipelineId)
 		return deployStageWfr, "", 0, err
@@ -1046,14 +1046,14 @@ func ReplaceImageTagWithDigest(image, digest string) string {
 }
 
 func (impl *HandlerServiceImpl) sendPreStageNotification(ctx context.Context, cdWf *pipelineConfig.CdWorkflow, pipeline *pipelineConfig.Pipeline) error {
-	wfr, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, cdWf.Id, bean2.CD_WORKFLOW_TYPE_PRE)
+	wfr, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, cdWf.Id, apiBean.CD_WORKFLOW_TYPE_PRE)
 	if err != nil {
 		return err
 	}
 
 	event, _ := impl.eventFactory.Build(util2.Trigger, &pipeline.Id, pipeline.AppId, &pipeline.EnvironmentId, util2.CD)
 	impl.logger.Debugw("event PreStageTrigger", "event", event)
-	event = impl.eventFactory.BuildExtraCDData(event, &wfr, 0, bean2.CD_WORKFLOW_TYPE_PRE)
+	event = impl.eventFactory.BuildExtraCDData(event, &wfr, 0, apiBean.CD_WORKFLOW_TYPE_PRE)
 	_, span := otel.Tracer("orchestrator").Start(ctx, "eventClient.WriteNotificationEvent")
 	_, evtErr := impl.eventClient.WriteNotificationEvent(event)
 	span.End()

pkg/executor/WorkflowService.go

@@ -20,7 +20,6 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
-	"fmt"
 	v1alpha12 "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned/typed/workflow/v1alpha1"
 	"github.com/argoproj/argo-workflows/v3/workflow/util"
 	"github.com/devtron-labs/common-lib/utils"
@@ -324,23 +323,16 @@ func (impl *WorkflowServiceImpl) prepareCmCsForWorkflowTemplate(workflowRequest
 		return nil, nil, err
 	}
 	allowAll := workflowRequest.IsDevtronJob() || workflowRequest.IsDevtronCI()
-	namePrefix := workflowRequest.GetExistingCmCsNamePrefix()
 	for _, cm := range workflowConfigMaps {
 		// HERE we are allowing all existingSecrets in case of JOB/ BUILD Infra
 		if _, ok := pipelineLevelConfigMaps[cm.Name]; ok || allowAll {
-			if !cm.External {
-				cm.Name = fmt.Sprintf("%s-cm-%s", cm.Name, namePrefix)
-			}
-			modifiedWorkflowConfigMaps = append(modifiedWorkflowConfigMaps, cm)
+			modifiedWorkflowConfigMaps = append(modifiedWorkflowConfigMaps, workflowRequest.ModifyConfigSecretMap(cm, bean.ConfigMap))
 		}
 	}
 	for _, secret := range workflowSecrets {
 		// HERE we are allowing all existingSecrets in case of JOB/ BUILD Infra
 		if _, ok := pipelineLevelSecrets[secret.Name]; ok || allowAll {
-			if !secret.External {
-				secret.Name = fmt.Sprintf("%s-cs-%s", secret.Name, namePrefix)
-			}
-			modifiedWorkflowSecrets = append(modifiedWorkflowSecrets, secret)
+			modifiedWorkflowSecrets = append(modifiedWorkflowSecrets, workflowRequest.ModifyConfigSecretMap(secret, bean.Secret))
 		}
 	}
 	return modifiedWorkflowConfigMaps, modifiedWorkflowSecrets, nil

pkg/pipeline/ArgoWorkflowExecutor_test.go

@@ -75,8 +75,8 @@ func TestExecuteWorkflow(t *testing.T) {
 		secretKeySecret := s3Artifact.SecretKeySecret
 		assert.NotNil(t, accessKeySecret)
 		assert.NotNil(t, secretKeySecret)
-		assert.True(t, reflect.DeepEqual(accessKeySecret, executors.ACCESS_KEY_SELECTOR))
-		assert.True(t, reflect.DeepEqual(secretKeySecret, executors.SECRET_KEY_SELECTOR))
+		assert.True(t, reflect.DeepEqual(accessKeySecret, executors.AccessKeySelector))
+		assert.True(t, reflect.DeepEqual(secretKeySecret, executors.SecretKeySelector))
 	})
 
 	t.Run("validate s3 blob storage with endpoint", func(t *testing.T) {
@@ -105,7 +105,7 @@ func TestExecuteWorkflow(t *testing.T) {
 		assert.Equal(t, gcpBlobStorage.LogBucketName, gcsArtifact.Bucket)
 		secretKeySecret := gcsArtifact.ServiceAccountKeySecret
 		assert.NotNil(t, secretKeySecret)
-		assert.True(t, reflect.DeepEqual(secretKeySecret, executors.SECRET_KEY_SELECTOR))
+		assert.True(t, reflect.DeepEqual(secretKeySecret, executors.SecretKeySelector))
 	})
 	t.Run("validate env specific cm and secret", func(t *testing.T) {
 		workflowTemplate := getBaseWorkflowTemplate(cdConfig)

pkg/pipeline/PipelineStageService.go

@@ -1985,14 +1985,14 @@ func (impl *PipelineStageServiceImpl) buildPipelineStepDataForWfRequest(step *re
 
 func (impl *PipelineStageServiceImpl) buildVariableAndConditionDataForWfRequest(stepId int) (*bean.VariableAndConditionDataForStep, error) {
 	variableAndConditionData := bean.NewVariableAndConditionDataForStep()
-	//getting all variables in the step
-	variables, err := impl.pipelineStageRepository.GetVariablesByStepId(stepId)
+	// getting all variables in the step
+	stepVariables, err := impl.pipelineStageRepository.GetVariablesByStepId(stepId)
 	if err != nil && !util.IsErrNoRows(err) {
 		impl.logger.Errorw("error in getting variables by stepId", "err", err, "stepId", stepId)
 		return variableAndConditionData, err
 	}
 	variableNameIdMap := make(map[int]string)
-	for _, variable := range variables {
+	for _, variable := range stepVariables {
 		variableNameIdMap[variable.Id] = variable.Name
 		// getting format
 		// ignoring error as it is already validated in func validatePipelineStageStepVariableForTrigger

pkg/pipeline/bean/pipelineStage.go

@@ -20,7 +20,7 @@ import (
 	commonBean "github.com/devtron-labs/common-lib/workflow"
 	"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig"
 	"github.com/devtron-labs/devtron/pkg/pipeline/repository"
-	repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository"
+	pluginRepo "github.com/devtron-labs/devtron/pkg/plugin/repository"
 	"github.com/devtron-labs/devtron/pkg/resourceQualifiers"
 )
 
@@ -46,23 +46,23 @@ type PipelineStageStepDto struct {
 }
 
 type InlineStepDetailDto struct {
-	ScriptType               repository2.ScriptType                `json:"scriptType" validate:"omitempty,oneof=SHELL DOCKERFILE CONTAINER_IMAGE"`
-	Script                   string                                `json:"script"`
-	StoreScriptAt            string                                `json:"storeScriptAt"`
-	DockerfileExists         bool                                  `json:"dockerfileExists,omitempty"`
-	MountPath                string                                `json:"mountPath,omitempty"`
-	MountCodeToContainer     bool                                  `json:"mountCodeToContainer,omitempty"`
-	MountCodeToContainerPath string                                `json:"mountCodeToContainerPath,omitempty"`
-	MountDirectoryFromHost   bool                                  `json:"mountDirectoryFromHost"`
-	ContainerImagePath       string                                `json:"containerImagePath,omitempty"`
-	ImagePullSecretType      repository2.ScriptImagePullSecretType `json:"imagePullSecretType,omitempty" validate:"omitempty,oneof=CONTAINER_REGISTRY SECRET_PATH"`
-	ImagePullSecret          string                                `json:"imagePullSecret,omitempty"`
-	MountPathMap             []*MountPathMap                       `json:"mountPathMap,omitempty" validate:"omitempty,dive"`
-	CommandArgsMap           []*CommandArgsMap                     `json:"commandArgsMap,omitempty" validate:"omitempty,dive"`
-	PortMap                  []*PortMap                            `json:"portMap,omitempty" validate:"omitempty,dive"`
-	InputVariables           []*StepVariableDto                    `json:"inputVariables" validate:"dive"`
-	OutputVariables          []*StepVariableDto                    `json:"outputVariables" validate:"dive"`
-	ConditionDetails         []*ConditionDetailDto                 `json:"conditionDetails" validate:"dive"`
+	ScriptType               pluginRepo.ScriptType                `json:"scriptType" validate:"omitempty,oneof=SHELL DOCKERFILE CONTAINER_IMAGE"`
+	Script                   string                               `json:"script"`
+	StoreScriptAt            string                               `json:"storeScriptAt"`
+	DockerfileExists         bool                                 `json:"dockerfileExists,omitempty"`
+	MountPath                string                               `json:"mountPath,omitempty"`
+	MountCodeToContainer     bool                                 `json:"mountCodeToContainer,omitempty"`
+	MountCodeToContainerPath string                               `json:"mountCodeToContainerPath,omitempty"`
+	MountDirectoryFromHost   bool                                 `json:"mountDirectoryFromHost"`
+	ContainerImagePath       string                               `json:"containerImagePath,omitempty"`
+	ImagePullSecretType      pluginRepo.ScriptImagePullSecretType `json:"imagePullSecretType,omitempty" validate:"omitempty,oneof=CONTAINER_REGISTRY SECRET_PATH"`
+	ImagePullSecret          string                               `json:"imagePullSecret,omitempty"`
+	MountPathMap             []*MountPathMap                      `json:"mountPathMap,omitempty" validate:"omitempty,dive"`
+	CommandArgsMap           []*CommandArgsMap                    `json:"commandArgsMap,omitempty" validate:"omitempty,dive"`
+	PortMap                  []*PortMap                           `json:"portMap,omitempty" validate:"omitempty,dive"`
+	InputVariables           []*StepVariableDto                   `json:"inputVariables" validate:"dive"`
+	OutputVariables          []*StepVariableDto                   `json:"outputVariables" validate:"dive"`
+	ConditionDetails         []*ConditionDetailDto                `json:"conditionDetails" validate:"dive"`
 }
 
 type RefPluginStepDetailDto struct {
@@ -92,9 +92,6 @@ type StepVariableDto struct {
 	StepVariableEntDto
 }
 
-type StepVariableEntDto struct {
-}
-
 func (s *StepVariableDto) GetValue() string {
 	if s == nil {
 		return ""
@@ -181,6 +178,7 @@ type VariableAndConditionDataForStep struct {
 	outputVariables          []*commonBean.VariableObject
 	triggerSkipConditions    []*ConditionObject
 	successFailureConditions []*ConditionObject
+	VariableAndConditionDataForStepEnt
 }
 
 func NewVariableAndConditionDataForStep() *VariableAndConditionDataForStep {

pkg/pipeline/bean/pipelineStage_ent.go

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2024. Devtron Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package bean
+
+import apiBean "github.com/devtron-labs/devtron/api/bean"
+
+type StepVariableEntDto struct {
+}
+
+type VariableAndConditionDataForStepEnt struct {
+}
+
+func (v *VariableAndConditionDataForStep) GetFileReferenceConfigMap() []*apiBean.ConfigSecretMap {
+	return nil
+}
+
+func (v *VariableAndConditionDataForStep) AddFileReferenceConfigMap(configMap *apiBean.ConfigSecretMap) *VariableAndConditionDataForStep {
+	return v
+}

pkg/pipeline/bean/workFlowRequestBean.go

@@ -42,6 +42,7 @@ type PrePostAndRefPluginStepsResponse struct {
 	PostStageSteps   []*StepObject
 	RefPluginData    []*RefPluginObject
 	VariableSnapshot map[string]string
+	PrePostAndRefPluginStepsResponseEnt
 }
 
 type StepObject struct {

pkg/pipeline/bean/workFlowRequestBean_ent.go

@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2024. Devtron Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package bean
+
+type PrePostAndRefPluginStepsResponseEnt struct {
+}