Merge pull request #6546 from devtron-labs/feat-file-variable-cm-mount-main

feat: mount file type variable in ci-runner through cm

Author: Ash-exp

api/bean/ConfigMapAndSecret.go

@@ -57,14 +57,19 @@ type ConfigSecretMap struct {
 	SubPath        bool            `json:"subPath"`
 	ESOSubPath     []string        `json:"esoSubPath"`
 	FilePermission string          `json:"filePermission"`
+	ConfigSecretMapEnt
 }
 
-func (configSecret ConfigSecretMap) GetDataMap() (map[string]string, error) {
-	var datamap map[string]string
-	err := json.Unmarshal(configSecret.Data, &datamap)
-	return datamap, err
+func (configSecret *ConfigSecretMap) GetDataMap() (map[string]string, error) {
+	if len(configSecret.Data) == 0 {
+		return make(map[string]string), nil
+	}
+	var dataMap map[string]string
+	err := json.Unmarshal(configSecret.Data, &dataMap)
+	return dataMap, err
 }
-func (configSecretJson ConfigSecretJson) GetDereferencedSecrets() []ConfigSecretMap {
+
+func (configSecretJson *ConfigSecretJson) GetDereferencedSecrets() []ConfigSecretMap {
 	return sliceUtil.GetDeReferencedSlice(configSecretJson.Secrets)
 }
 
@@ -95,3 +100,14 @@ func GetTransformedDataForSecretRootJsonData(data string, mode util.SecretTransf
 	}
 	return string(marshal), nil
 }
+
+type ConfigType string
+
+func (c ConfigType) String() string {
+	return string(c)
+}
+
+const (
+	ConfigMap ConfigType = "cm"
+	Secret    ConfigType = "cs"
+)

api/bean/ConfigMapAndSecret_ent.go

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2024. Devtron Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package bean
+
+type ConfigSecretMapEnt struct {
+}
+
+func (configSecret *ConfigSecretMap) AddDataToKey(keyName string, data []byte) (*ConfigSecretMap, error) {
+	return configSecret, nil
+}
+
+func (configSecret *ConfigSecretMap) GetBinaryDataMap() map[string][]byte {
+	return nil
+}

go.mod

@@ -307,8 +307,8 @@ require (
 
 replace (
 	github.com/argoproj/argo-workflows/v3 v3.5.13 => github.com/devtron-labs/argo-workflows/v3 v3.5.13
-	github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250425083942-5092ec30954c
-	github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425083942-5092ec30954c
+	github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250502103233-29d5c52295e4
+	github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250502103233-29d5c52295e4
 	github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127
 	github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5
 	k8s.io/api => k8s.io/api v0.29.7

go.sum

@@ -829,10 +829,10 @@ github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzq
 github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/devtron-labs/argo-workflows/v3 v3.5.13 h1:3pINq0gXOSeTw2z/vYe+j80lRpSN5Rp/8mfQORh8SmU=
 github.com/devtron-labs/argo-workflows/v3 v3.5.13/go.mod h1:/vqxcovDPT4zqr4DjR5v7CF8ggpY1l3TSa2CIG3jmjA=
-github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250425083942-5092ec30954c h1:UZq25z5WE7MQZAXe6uvM/xNektllueo4mz4l4943raU=
-github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250425083942-5092ec30954c/go.mod h1:FfaLDXN1ZXxyRpnskBqVIYkpkWDCzBmDgIO9xqLnxdQ=
-github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425083942-5092ec30954c h1:lIIGAMWM4UwSwrd5Ljm1x4vtNRm1s0q2mTqdQWX8L6Q=
-github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425083942-5092ec30954c/go.mod h1:zkNShlkcHxsmnL0gKNbs0uyRL8lZonGKr5Km63uTLI0=
+github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250502103233-29d5c52295e4 h1:FZ3z1sbg2qyconeuDSJKsku56gTKtKsEeodagDIpQq8=
+github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250502103233-29d5c52295e4/go.mod h1:FfaLDXN1ZXxyRpnskBqVIYkpkWDCzBmDgIO9xqLnxdQ=
+github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250502103233-29d5c52295e4 h1:Y66AO2dPjzFC+AbEkaHP3jGV15x9X4ALfDVZ1SNs/e4=
+github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250502103233-29d5c52295e4/go.mod h1:zkNShlkcHxsmnL0gKNbs0uyRL8lZonGKr5Km63uTLI0=
 github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU=
 github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y=
 github.com/devtron-labs/protos v0.0.3-0.20250323220609-ecf8a0f7305e h1:U6UdYbW8a7xn5IzFPd8cywjVVPfutGJCudjePAfL/Hs=

pkg/deployment/trigger/devtronApps/preStageHandlerCode.go

@@ -23,7 +23,7 @@ import (
 	"github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
 	blob_storage "github.com/devtron-labs/common-lib/blob-storage"
 	commonBean "github.com/devtron-labs/common-lib/workflow"
-	bean2 "github.com/devtron-labs/devtron/api/bean"
+	apiBean "github.com/devtron-labs/devtron/api/bean"
 	gitSensorClient "github.com/devtron-labs/devtron/client/gitSensor"
 	constants2 "github.com/devtron-labs/devtron/internal/sql/constants"
 	"github.com/devtron-labs/devtron/internal/sql/repository"
@@ -62,7 +62,7 @@ import (
 )
 
 func (impl *HandlerServiceImpl) TriggerPreStage(request bean.TriggerRequest) (*bean6.ManifestPushTemplate, error) {
-	request.WorkflowType = bean2.CD_WORKFLOW_TYPE_PRE
+	request.WorkflowType = apiBean.CD_WORKFLOW_TYPE_PRE
 	// setting triggeredAt variable to have consistent data for various audit log places in db for deployment time
 	triggeredAt := time.Now()
 	triggeredBy := request.TriggeredBy
@@ -219,7 +219,7 @@ func (impl *HandlerServiceImpl) TriggerAutoCDOnPreStageSuccess(triggerContext be
 func (impl *HandlerServiceImpl) checkDeploymentTriggeredAlready(wfId int) bool {
 	deploymentTriggeredAlready := false
 	// TODO : need to check this logic for status check in case of multiple deployments requirement for same workflow
-	workflowRunner, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(context.Background(), wfId, bean2.CD_WORKFLOW_TYPE_DEPLOY)
+	workflowRunner, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(context.Background(), wfId, apiBean.CD_WORKFLOW_TYPE_DEPLOY)
 	if err != nil {
 		impl.logger.Errorw("error occurred while fetching workflow runner", "wfId", wfId, "err", err)
 		return deploymentTriggeredAlready
@@ -236,7 +236,7 @@ func (impl *HandlerServiceImpl) createStartingWfAndRunner(request bean.TriggerRe
 	//in case of pre stage manual trigger auth is already applied and for auto triggers there is no need for auth check here
 	cdWf := request.CdWf
 	var err error
-	if cdWf == nil && request.WorkflowType == bean2.CD_WORKFLOW_TYPE_PRE {
+	if cdWf == nil && request.WorkflowType == apiBean.CD_WORKFLOW_TYPE_PRE {
 		cdWf = &pipelineConfig.CdWorkflow{
 			CiArtifactId: artifact.Id,
 			PipelineId:   pipeline.Id,
@@ -279,9 +279,9 @@ func (impl *HandlerServiceImpl) getEnvAndNsIfRunStageInEnv(ctx context.Context,
 	var err error
 	namespace := impl.config.GetDefaultNamespace()
 	runStageInEnv := false
-	if workflowStage == bean2.CD_WORKFLOW_TYPE_PRE {
+	if workflowStage == apiBean.CD_WORKFLOW_TYPE_PRE {
 		runStageInEnv = pipeline.RunPreStageInEnv
-	} else if workflowStage == bean2.CD_WORKFLOW_TYPE_POST {
+	} else if workflowStage == apiBean.CD_WORKFLOW_TYPE_POST {
 		runStageInEnv = pipeline.RunPostStageInEnv
 	}
 	_, span := otel.Tracer("orchestrator").Start(ctx, "envRepository.FindById")
@@ -588,7 +588,7 @@ func (impl *HandlerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow
 	}
 	if pipelineStage != nil {
 		var variableSnapshot map[string]string
-		if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_PRE {
+		if runner.WorkflowType == apiBean.CD_WORKFLOW_TYPE_PRE {
 			// TODO: use const from pipeline.WorkflowService:95
 			request := pipelineConfigBean.NewBuildPrePostStepDataReq(cdPipeline.Id, "preCD", scope)
 			prePostAndRefPluginResponse, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(request)
@@ -599,7 +599,7 @@ func (impl *HandlerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow
 			preDeploySteps = prePostAndRefPluginResponse.PreStageSteps
 			refPluginsData = prePostAndRefPluginResponse.RefPluginData
 			variableSnapshot = prePostAndRefPluginResponse.VariableSnapshot
-		} else if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_POST {
+		} else if runner.WorkflowType == apiBean.CD_WORKFLOW_TYPE_POST {
 			// TODO: use const from pipeline.WorkflowService:96
 			request := pipelineConfigBean.NewBuildPrePostStepDataReq(cdPipeline.Id, "postCD", scope)
 			prePostAndRefPluginResponse, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(request)
@@ -630,9 +630,9 @@ func (impl *HandlerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow
 		}
 	} else {
 		//in this case no plugin script is not present for this cdPipeline hence going with attaching preStage or postStage config
-		if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_PRE {
+		if runner.WorkflowType == apiBean.CD_WORKFLOW_TYPE_PRE {
 			stageYaml = cdPipeline.PreStageConfig
-		} else if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_POST {
+		} else if runner.WorkflowType == apiBean.CD_WORKFLOW_TYPE_POST {
 			stageYaml = cdPipeline.PostStageConfig
 			deployStageWfr, deployStageTriggeredByUserEmail, pipelineReleaseCounter, err = impl.getDeployStageDetails(cdPipeline.Id)
 			if err != nil {
@@ -813,11 +813,11 @@ func (impl *HandlerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow
 	// For Pre-CD / Post-CD workflow, cache is not uploaded; hence no need to set cache bucket
 	cdWorkflowConfigCdCacheBucket := ""
 
-	if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_PRE {
+	if runner.WorkflowType == apiBean.CD_WORKFLOW_TYPE_PRE {
 		// populate input variables of steps with extra env variables
 		setExtraEnvVariableInDeployStep(preDeploySteps, runtimeParams.GetSystemVariables(), webhookAndCiData)
 		cdStageWorkflowRequest.PrePostDeploySteps = preDeploySteps
-	} else if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_POST {
+	} else if runner.WorkflowType == apiBean.CD_WORKFLOW_TYPE_POST {
 		// populate input variables of steps with extra env variables
 		setExtraEnvVariableInDeployStep(postDeploySteps, runtimeParams.GetSystemVariables(), webhookAndCiData)
 		cdStageWorkflowRequest.PrePostDeploySteps = postDeploySteps
@@ -1007,7 +1007,7 @@ func setExtraEnvVariableInDeployStep(deploySteps []*pipelineConfigBean.StepObjec
 func (impl *HandlerServiceImpl) getDeployStageDetails(pipelineId int) (pipelineConfig.CdWorkflowRunner, string, int, error) {
 	deployStageWfr := pipelineConfig.CdWorkflowRunner{}
 	//getting deployment pipeline latest wfr by pipelineId
-	deployStageWfr, err := impl.cdWorkflowRepository.FindLatestByPipelineIdAndRunnerType(pipelineId, bean2.CD_WORKFLOW_TYPE_DEPLOY)
+	deployStageWfr, err := impl.cdWorkflowRepository.FindLatestByPipelineIdAndRunnerType(pipelineId, apiBean.CD_WORKFLOW_TYPE_DEPLOY)
 	if err != nil {
 		impl.logger.Errorw("error in getting latest status of deploy type wfr by pipelineId", "err", err, "pipelineId", pipelineId)
 		return deployStageWfr, "", 0, err
@@ -1046,14 +1046,14 @@ func ReplaceImageTagWithDigest(image, digest string) string {
 }
 
 func (impl *HandlerServiceImpl) sendPreStageNotification(ctx context.Context, cdWf *pipelineConfig.CdWorkflow, pipeline *pipelineConfig.Pipeline) error {
-	wfr, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, cdWf.Id, bean2.CD_WORKFLOW_TYPE_PRE)
+	wfr, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, cdWf.Id, apiBean.CD_WORKFLOW_TYPE_PRE)
 	if err != nil {
 		return err
 	}
 
 	event, _ := impl.eventFactory.Build(util2.Trigger, &pipeline.Id, pipeline.AppId, &pipeline.EnvironmentId, util2.CD)
 	impl.logger.Debugw("event PreStageTrigger", "event", event)
-	event = impl.eventFactory.BuildExtraCDData(event, &wfr, 0, bean2.CD_WORKFLOW_TYPE_PRE)
+	event = impl.eventFactory.BuildExtraCDData(event, &wfr, 0, apiBean.CD_WORKFLOW_TYPE_PRE)
 	_, span := otel.Tracer("orchestrator").Start(ctx, "eventClient.WriteNotificationEvent")
 	_, evtErr := impl.eventClient.WriteNotificationEvent(event)
 	span.End()

pkg/executor/WorkflowService.go

@@ -20,7 +20,6 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
-	"fmt"
 	v1alpha12 "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned/typed/workflow/v1alpha1"
 	"github.com/argoproj/argo-workflows/v3/workflow/util"
 	"github.com/devtron-labs/common-lib/utils"
@@ -324,23 +323,16 @@ func (impl *WorkflowServiceImpl) prepareCmCsForWorkflowTemplate(workflowRequest
 		return nil, nil, err
 	}
 	allowAll := workflowRequest.IsDevtronJob() || workflowRequest.IsDevtronCI()
-	namePrefix := workflowRequest.GetExistingCmCsNamePrefix()
 	for _, cm := range workflowConfigMaps {
 		// HERE we are allowing all existingSecrets in case of JOB/ BUILD Infra
 		if _, ok := pipelineLevelConfigMaps[cm.Name]; ok || allowAll {
-			if !cm.External {
-				cm.Name = fmt.Sprintf("%s-cm-%s", cm.Name, namePrefix)
-			}
-			modifiedWorkflowConfigMaps = append(modifiedWorkflowConfigMaps, cm)
+			modifiedWorkflowConfigMaps = append(modifiedWorkflowConfigMaps, workflowRequest.ModifyConfigSecretMap(cm, bean.ConfigMap))
 		}
 	}
 	for _, secret := range workflowSecrets {
 		// HERE we are allowing all existingSecrets in case of JOB/ BUILD Infra
 		if _, ok := pipelineLevelSecrets[secret.Name]; ok || allowAll {
-			if !secret.External {
-				secret.Name = fmt.Sprintf("%s-cs-%s", secret.Name, namePrefix)
-			}
-			modifiedWorkflowSecrets = append(modifiedWorkflowSecrets, secret)
+			modifiedWorkflowSecrets = append(modifiedWorkflowSecrets, workflowRequest.ModifyConfigSecretMap(secret, bean.Secret))
 		}
 	}
 	return modifiedWorkflowConfigMaps, modifiedWorkflowSecrets, nil

pkg/pipeline/ArgoWorkflowExecutor_test.go

@@ -75,8 +75,8 @@ func TestExecuteWorkflow(t *testing.T) {
 		secretKeySecret := s3Artifact.SecretKeySecret
 		assert.NotNil(t, accessKeySecret)
 		assert.NotNil(t, secretKeySecret)
-		assert.True(t, reflect.DeepEqual(accessKeySecret, executors.ACCESS_KEY_SELECTOR))
-		assert.True(t, reflect.DeepEqual(secretKeySecret, executors.SECRET_KEY_SELECTOR))
+		assert.True(t, reflect.DeepEqual(accessKeySecret, executors.AccessKeySelector))
+		assert.True(t, reflect.DeepEqual(secretKeySecret, executors.SecretKeySelector))
 	})
 
 	t.Run("validate s3 blob storage with endpoint", func(t *testing.T) {
@@ -105,7 +105,7 @@ func TestExecuteWorkflow(t *testing.T) {
 		assert.Equal(t, gcpBlobStorage.LogBucketName, gcsArtifact.Bucket)
 		secretKeySecret := gcsArtifact.ServiceAccountKeySecret
 		assert.NotNil(t, secretKeySecret)
-		assert.True(t, reflect.DeepEqual(secretKeySecret, executors.SECRET_KEY_SELECTOR))
+		assert.True(t, reflect.DeepEqual(secretKeySecret, executors.SecretKeySelector))
 	})
 	t.Run("validate env specific cm and secret", func(t *testing.T) {
 		workflowTemplate := getBaseWorkflowTemplate(cdConfig)

pkg/pipeline/PipelineStageService.go

@@ -1985,14 +1985,14 @@ func (impl *PipelineStageServiceImpl) buildPipelineStepDataForWfRequest(step *re
 
 func (impl *PipelineStageServiceImpl) buildVariableAndConditionDataForWfRequest(stepId int) (*bean.VariableAndConditionDataForStep, error) {
 	variableAndConditionData := bean.NewVariableAndConditionDataForStep()
-	//getting all variables in the step
-	variables, err := impl.pipelineStageRepository.GetVariablesByStepId(stepId)
+	// getting all variables in the step
+	stepVariables, err := impl.pipelineStageRepository.GetVariablesByStepId(stepId)
 	if err != nil && !util.IsErrNoRows(err) {
 		impl.logger.Errorw("error in getting variables by stepId", "err", err, "stepId", stepId)
 		return variableAndConditionData, err
 	}
 	variableNameIdMap := make(map[int]string)
-	for _, variable := range variables {
+	for _, variable := range stepVariables {
 		variableNameIdMap[variable.Id] = variable.Name
 		// getting format
 		// ignoring error as it is already validated in func validatePipelineStageStepVariableForTrigger