diff --git a/docs/user-guide/global-configurations/build-infra.md b/docs/user-guide/global-configurations/build-infra.md index 2b1ea6b11f..027c82b4be 100644 --- a/docs/user-guide/global-configurations/build-infra.md +++ b/docs/user-guide/global-configurations/build-infra.md @@ -117,6 +117,27 @@ If you delete a profile attached to one or more applications, the [default profi If you need extra control on the build infra configuration apart from CPU, memory, and build timeout, feel free to open a [GitHub issue](https://github.com/devtron-labs/devtron/issues) for us to help you. +### Pipeline-Level Build Infra Selection + +In addition to application-level build infra profiles, you can now assign specific build infrastructure profiles at the individual CI pipeline level. This provides more granular control for scenarios where: + +1. **Production pipelines** require different build infrastructure than development pipelines +2. **Different build types** (e.g., security scanning vs. fast builds) need specific resource configurations +3. **Environment-specific builds** require tailored infrastructure settings + +**How it works:** +- When creating or editing a CI pipeline, you can optionally select a specific build infra profile +- If a pipeline-level profile is selected, it takes priority over the application-level profile +- If no pipeline-level profile is specified, the system falls back to the application-level profile +- This maintains full backward compatibility with existing configurations + +**Use cases:** +- Assign high-resource profiles to production CI pipelines while keeping development pipelines on standard resources +- Use specialized build infrastructure for pipelines that include security scanning or extensive testing +- Optimize costs by using appropriate infrastructure sizes for different types of builds + +This feature allows you to define build infrastructure for all production pipelines (CI pipelines attached to production environments) while providing flexibility to pass node-level selectors for development builds as needed. + --- ## Extras diff --git a/internal/sql/repository/pipelineConfig/CiPipelineRepository.go b/internal/sql/repository/pipelineConfig/CiPipelineRepository.go index c6adb8b7a8..86a342bec3 100644 --- a/internal/sql/repository/pipelineConfig/CiPipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/CiPipelineRepository.go @@ -50,6 +50,7 @@ type CiPipeline struct { ScanEnabled bool `sql:"scan_enabled,notnull"` IsDockerConfigOverridden bool `sql:"is_docker_config_overridden, notnull"` PipelineType string `sql:"ci_pipeline_type"` + InfraProfileId *int `sql:"infra_profile_id"` // Optional pipeline-level infra profile sql.AuditLog CiPipelineMaterials []*CiPipelineMaterial CiTemplate *CiTemplate @@ -146,6 +147,8 @@ type CiPipelineRepository interface { GetLinkedCiPipelines(ctx context.Context, ciPipelineId int) ([]*CiPipeline, error) GetDownStreamInfo(ctx context.Context, sourceCiPipelineId int, appNameMatch, envNameMatch string, req *pagination.RepositoryRequest) ([]ciPipeline.LinkedCIDetails, int, error) + // GetInfraProfileIdByPipelineId gets the infra profile ID assigned to a specific CI pipeline + GetInfraProfileIdByPipelineId(pipelineId int) (*int, error) } type CiPipelineRepositoryImpl struct { @@ -726,3 +729,19 @@ func (impl *CiPipelineRepositoryImpl) GetDownStreamInfo(ctx context.Context, sou } return linkedCIDetails, totalCount, err } + +func (impl *CiPipelineRepositoryImpl) GetInfraProfileIdByPipelineId(pipelineId int) (*int, error) { + var infraProfileId *int + err := impl.dbConnection.Model((*CiPipeline)(nil)). + Where("id = ?", pipelineId). + Where("deleted = ?", false). + Column("infra_profile_id"). + Select(&infraProfileId) + if err != nil { + if errors.Is(err, pg.ErrNoRows) { + return nil, nil + } + return nil, err + } + return infraProfileId, nil +} diff --git a/pkg/bean/app.go b/pkg/bean/app.go index d198f849c2..f4e8240de5 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -151,6 +151,7 @@ type CiPipeline struct { CustomTagObject *CustomTagData `json:"customTag,omitempty"` DefaultTag []string `json:"defaultTag,omitempty"` EnableCustomTag bool `json:"enableCustomTag"` + InfraProfileId *int `json:"infraProfileId,omitempty"` // Optional pipeline-level infra profile override } func (ciPipeline *CiPipeline) IsLinkedCi() bool { diff --git a/pkg/build/trigger/HandlerService.go b/pkg/build/trigger/HandlerService.go index 0e82e62cd2..72103eef15 100644 --- a/pkg/build/trigger/HandlerService.go +++ b/pkg/build/trigger/HandlerService.go @@ -772,7 +772,8 @@ func (impl *HandlerServiceImpl) StartCiWorkflowAndPrepareWfRequest(trigger *type } scope := resourceQualifiers.Scope{ - AppId: pipeline.App.Id, + AppId: pipeline.App.Id, + PipelineId: pipeline.Id, // Added pipeline ID for pipeline-level infra profile support } ciWorkflowConfigNamespace := impl.config.GetDefaultNamespace() envModal, isJob, err := impl.getEnvironmentForJob(pipeline, trigger) diff --git a/pkg/infraConfig/service/infraConfigService_ent.go b/pkg/infraConfig/service/infraConfigService_ent.go index f8105b05a2..d2145288a2 100644 --- a/pkg/infraConfig/service/infraConfigService_ent.go +++ b/pkg/infraConfig/service/infraConfigService_ent.go @@ -21,6 +21,7 @@ import ( "github.com/devtron-labs/devtron/pkg/infraConfig/repository" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" "github.com/go-pg/pg" + "github.com/pkg/errors" ) type InfraConfigServiceEnt interface { @@ -51,6 +52,34 @@ func (impl *InfraConfigServiceImpl) getDefaultBuildxDriverType() v1.BuildxDriver } func (impl *InfraConfigServiceImpl) getInfraProfileIdsByScope(scope *v1.Scope) ([]int, error) { - // for OSS, user can't create infra profiles so no need to fetch infra profiles + // First check if there's a pipeline-level infra profile + if scope.PipelineId > 0 { + pipelineInfraProfileId, err := impl.getPipelineInfraProfileId(scope.PipelineId) + if err != nil { + impl.logger.Errorw("error in fetching pipeline-level infra profile", "pipelineId", scope.PipelineId, "error", err) + return make([]int, 0), err + } + if pipelineInfraProfileId != nil { + impl.logger.Debugw("found pipeline-level infra profile", "pipelineId", scope.PipelineId, "infraProfileId", *pipelineInfraProfileId) + return []int{*pipelineInfraProfileId}, nil + } + } + + // For OSS, user can't create infra profiles so no need to fetch infra profiles + // Falls back to global profile return make([]int, 0), nil } + +func (impl *InfraConfigServiceImpl) getPipelineInfraProfileId(pipelineId int) (*int, error) { + // Query the ci_pipeline table directly for the infra_profile_id + var infraProfileId *int + query := `SELECT infra_profile_id FROM ci_pipeline WHERE id = ? AND deleted = false` + _, err := impl.infraProfileRepo.GetDbConnection().Query(&infraProfileId, query, pipelineId) + if err != nil { + if errors.Is(err, pg.ErrNoRows) { + return nil, nil // Pipeline not found, fall back to app-level profile + } + return nil, err + } + return infraProfileId, nil +} diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index 3d8750ea81..18c5ae0af0 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -403,6 +403,7 @@ func (impl CiCdPipelineOrchestratorImpl) PatchMaterialValue(createRequest *bean. ParentCiPipeline: createRequest.ParentCiPipeline, ScanEnabled: createRequest.ScanEnabled, IsDockerConfigOverridden: createRequest.IsDockerConfigOverridden, + InfraProfileId: createRequest.InfraProfileId, // Pipeline-level infra profile support AuditLog: sql.AuditLog{UpdatedBy: userId, UpdatedOn: time.Now()}, } @@ -949,6 +950,7 @@ func (impl CiCdPipelineOrchestratorImpl) CreateCiConf(createRequest *bean.CiConf ScanEnabled: createRequest.ScanEnabled, IsDockerConfigOverridden: ciPipeline.IsDockerConfigOverridden, PipelineType: string(ciPipeline.PipelineType), + InfraProfileId: ciPipeline.InfraProfileId, // Pipeline-level infra profile support AuditLog: sql.AuditLog{UpdatedBy: createRequest.UserId, CreatedBy: createRequest.UserId, UpdatedOn: time.Now(), CreatedOn: time.Now()}, } err = impl.ciPipelineRepository.Save(ciPipelineObject, tx) diff --git a/scripts/sql/34604300_pipeline_infra_profile.down.sql b/scripts/sql/34604300_pipeline_infra_profile.down.sql new file mode 100644 index 0000000000..dd45b4cde5 --- /dev/null +++ b/scripts/sql/34604300_pipeline_infra_profile.down.sql @@ -0,0 +1,11 @@ +-- Rollback migration for pipeline-level infra profile support + +BEGIN; + +-- Drop the foreign key constraint +ALTER TABLE ci_pipeline DROP CONSTRAINT IF EXISTS fk_ci_pipeline_infra_profile; + +-- Drop the infra_profile_id column +ALTER TABLE ci_pipeline DROP COLUMN IF EXISTS infra_profile_id; + +COMMIT; \ No newline at end of file diff --git a/scripts/sql/34604300_pipeline_infra_profile.up.sql b/scripts/sql/34604300_pipeline_infra_profile.up.sql new file mode 100644 index 0000000000..2d21b3811e --- /dev/null +++ b/scripts/sql/34604300_pipeline_infra_profile.up.sql @@ -0,0 +1,18 @@ +-- Migration to add pipeline-level infra profile support to ci_pipeline table +-- This enables selecting different build infrastructure per pipeline + +BEGIN; + +-- Add infra_profile_id column to ci_pipeline table +ALTER TABLE ci_pipeline +ADD COLUMN IF NOT EXISTS infra_profile_id INTEGER; + +-- Add foreign key constraint to infra_profile table +ALTER TABLE ci_pipeline +ADD CONSTRAINT fk_ci_pipeline_infra_profile +FOREIGN KEY (infra_profile_id) REFERENCES infra_profile(id); + +-- Add comment to explain the column +COMMENT ON COLUMN ci_pipeline.infra_profile_id IS 'Optional pipeline-level infra profile override. If null, falls back to application-level profile.'; + +COMMIT; \ No newline at end of file