feat: add branch filtering for Jenkins jobs

Author: GoSimplicity

backend/plugins/jenkins/models/migrationscripts/20251002_add_branch_filter_pattern.go

@@ -0,0 +1,46 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+type addBranchFilterPattern struct{}
+
+type JenkinsScopeConfig20251002 struct {
+	BranchFilterPattern string `gorm:"type:varchar(255)"`
+}
+
+func (JenkinsScopeConfig20251002) TableName() string {
+	return "_tool_jenkins_scope_configs"
+}
+
+func (u *addBranchFilterPattern) Up(baseRes context.BasicRes) errors.Error {
+	return migrationhelper.AutoMigrateTables(baseRes, &JenkinsScopeConfig20251002{})
+}
+
+func (*addBranchFilterPattern) Version() uint64 {
+	return 20251002100000
+}
+
+func (*addBranchFilterPattern) Name() string {
+	return "add branch filter pattern to jenkins scope config"
+}

backend/plugins/jenkins/models/migrationscripts/register.go

@@ -36,5 +36,6 @@ func All() []plugin.MigrationScript {
 		new(renameTr2ScopeConfig),
 		new(addRawParamTableForScope),
 		new(addNumberToJenkinsBuildCommit),
+		new(addBranchFilterPattern),
 	}
 }

backend/plugins/jenkins/models/scope_config.go

@@ -22,9 +22,10 @@ import (
 )
 
 type JenkinsScopeConfig struct {
-	common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"`
-	DeploymentPattern  string `gorm:"type:varchar(255)" mapstructure:"deploymentPattern,omitempty" json:"deploymentPattern"`
-	ProductionPattern  string `gorm:"type:varchar(255)" mapstructure:"productionPattern,omitempty" json:"productionPattern"`
+	common.ScopeConfig  `mapstructure:",squash" json:",inline" gorm:"embedded"`
+	DeploymentPattern   string `mapstructure:"deploymentPattern,omitempty" json:"deploymentPattern" gorm:"type:varchar(255)"`
+	ProductionPattern   string `mapstructure:"productionPattern,omitempty" json:"productionPattern" gorm:"type:varchar(255)"`
+	BranchFilterPattern string `mapstructure:"branchFilterPattern,omitempty" json:"branchFilterPattern" gorm:"type:varchar(255)"`
 }
 
 func (t JenkinsScopeConfig) TableName() string {

backend/plugins/jenkins/tasks/job_collector.go

@@ -22,9 +22,11 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"regexp"
 	"time"
 
 	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/log"
 	"github.com/apache/incubator-devlake/core/plugin"
 	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 )
@@ -66,16 +68,26 @@ func CollectApiJobs(taskCtx plugin.SubTaskContext) errors.Error {
 					return query, nil
 				},
 				ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
-					var data struct {
+					var resData struct {
 						Jobs []json.RawMessage `json:"jobs"`
 					}
-					err := helper.UnmarshalResponse(res, &data)
+					err := helper.UnmarshalResponse(res, &resData)
 					if err != nil {
 						return nil, err
 					}
 
-					jobs := make([]json.RawMessage, 0, len(data.Jobs))
-					for _, job := range data.Jobs {
+					// Compile branch filter pattern once for this batch
+					var branchPattern *regexp.Regexp
+					if data.Options.ScopeConfig != nil && data.Options.ScopeConfig.BranchFilterPattern != "" {
+						var compileErr error
+						branchPattern, compileErr = regexp.Compile(data.Options.ScopeConfig.BranchFilterPattern)
+						if compileErr != nil {
+							logger.Warn(nil, "Invalid branch filter pattern: %s, will include all jobs", data.Options.ScopeConfig.BranchFilterPattern)
+						}
+					}
+
+					jobs := make([]json.RawMessage, 0, len(resData.Jobs))
+					for _, job := range resData.Jobs {
 						var jobObj map[string]interface{}
 						err := json.Unmarshal(job, &jobObj)
 						if err != nil {
@@ -84,7 +96,10 @@ func CollectApiJobs(taskCtx plugin.SubTaskContext) errors.Error {
 
 						logger.Debug("%v", jobObj)
 						if jobObj["color"] != "notbuilt" && jobObj["color"] != "nobuilt_anime" {
-							jobs = append(jobs, job)
+							// Apply branch filter pattern if configured
+							if shouldIncludeJob(jobObj, branchPattern, logger) {
+								jobs = append(jobs, job)
+							}
 						}
 					}
 
@@ -100,3 +115,26 @@ func CollectApiJobs(taskCtx plugin.SubTaskContext) errors.Error {
 
 	return collector.Execute()
 }
+
+// shouldIncludeJob determines whether a job should be included based on the branch filter pattern
+func shouldIncludeJob(jobObj map[string]interface{}, branchPattern *regexp.Regexp, logger log.Logger) bool {
+	// If no branch filter pattern is configured, include all jobs
+	if branchPattern == nil {
+		return true
+	}
+
+	// Get the job name for pattern matching
+	jobName, ok := jobObj["name"].(string)
+	if !ok {
+		// If we can't get the job name, include it by default
+		logger.Warn(nil, "Could not extract job name for filtering, including job by default")
+		return true
+	}
+
+	// Match the job name against the pattern
+	matched := branchPattern.MatchString(jobName)
+	logger.Debug("Job '%s' %s branch filter pattern", jobName,
+		map[bool]string{true: "matches", false: "does not match"}[matched])
+
+	return matched
+}

config-ui/src/plugins/register/jenkins/config.tsx

@@ -59,6 +59,7 @@ export const JenkinsConfig: IPluginConfig = {
     transformation: {
       deploymentPattern: '(deploy|push-image)',
       productionPattern: 'prod(.*)',
+      branchFilterPattern: '',
     },
   },
 };

config-ui/src/plugins/register/jenkins/transformation.tsx

@@ -74,6 +74,38 @@ const renderCollapseItems = ({
       children: (
         <>
           <h3 style={{ marginBottom: 16 }}>
+            <span>Branch Filtering</span>
+            <Tag style={{ marginLeft: 4 }} color="green">
+              NEW
+            </Tag>
+          </h3>
+          <p style={{ marginBottom: 16 }}>
+            Use Regular Expression to filter Jenkins jobs by branch name. This helps exclude temporary branch/PR builds from collection.{' '}
+            <ExternalLink link={DOC_URL.PLUGIN.JENKINS.TRANSFORMATION}>Learn more</ExternalLink>
+          </p>
+          <div style={{ marginTop: 16 }}>Only collect Jenkins jobs when: </div>
+          <div style={{ margin: '8px 0', paddingLeft: 28 }}>
+            <span>
+              The <strong>job name</strong> matches
+            </span>
+            <Input
+              style={{ width: 200, margin: '0 8px' }}
+              placeholder="^(main|master|develop).*"
+              value={transformation.branchFilterPattern ?? ''}
+              onChange={(e) =>
+                onChangeTransformation({
+                  ...transformation,
+                  branchFilterPattern: e.target.value,
+                })
+              }
+            />
+            <HelpTooltip content="Regular expression to match job names. Leave empty to collect all jobs. Example: '^(main|master|develop).*' to collect only main branches" />
+          </div>
+          <div style={{ margin: '8px 0', paddingLeft: 28, fontSize: '12px', color: '#666' }}>
+            <span>Examples: <code>^(main|master|develop).*</code> (main branches only), <code>.*production.*</code> (production jobs)</span>
+          </div>
+
+          <h3 style={{ marginBottom: 16, marginTop: 32 }}>
             <span>Deployment</span>
             <Tag style={{ marginLeft: 4 }} color="blue">
               DORA
@@ -117,7 +149,7 @@ const renderCollapseItems = ({
                 })
               }
             />
-            <span>, this Deployment is a ‘Production Deployment’</span>
+            <span>, this Deployment is a 'Production Deployment'</span>
             <HelpTooltip content="If you leave this field empty, all DevLake Deployments will be tagged as in the Production environment. " />
           </div>
         </>