Skip to content

Commit d2b96cc

Browse files
springdospringdo
authored
Merge pull request #25 from rht-labs/feature/multibranch-github
🕺 ADD - multibranch for github to dsl 🕺
2 parents 32143ab + 4ef0fef commit d2b96cc

File tree

3 files changed

+236
-53
lines changed

3 files changed

+236
-53
lines changed

‎configuration/jobs/seed-multibranch-pipelines/config.xml‎

Lines changed: 110 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,15 @@
1717
<artifactNumToKeep>-1</artifactNumToKeep>
1818
</strategy>
1919
</jenkins.model.BuildDiscarderProperty>
20-
<com.sonyericsson.jenkins.plugins.bfa.model.ScannerJobProperty plugin="build-failure-analyzer@1.20.0">
20+
<com.sonyericsson.jenkins.plugins.bfa.model.ScannerJobProperty plugin="build-failure-analyzer@1.26.0">
2121
<doNotScan>false</doNotScan>
2222
</com.sonyericsson.jenkins.plugins.bfa.model.ScannerJobProperty>
23-
<com.dabsquared.gitlabjenkins.connection.GitLabConnectionProperty plugin="[email protected].9">
23+
<com.dabsquared.gitlabjenkins.connection.GitLabConnectionProperty plugin="[email protected].13">
2424
<gitLabConnection></gitLabConnection>
2525
</com.dabsquared.gitlabjenkins.connection.GitLabConnectionProperty>
26-
<org.jenkinsci.plugins.gitlablogo.GitlabLogoProperty plugin="[email protected].3">
26+
<org.jenkinsci.plugins.gitlablogo.GitlabLogoProperty plugin="[email protected].5">
2727
<repositoryName></repositoryName>
2828
</org.jenkinsci.plugins.gitlablogo.GitlabLogoProperty>
29-
<org.jenkinsci.plugins.gogs.GogsProjectProperty plugin="[email protected]">
30-
<gogsSecret></gogsSecret>
31-
<gogsUsePayload>false</gogsUsePayload>
32-
</org.jenkinsci.plugins.gogs.GogsProjectProperty>
3329
<com.sonyericsson.rebuild.RebuildSettings plugin="[email protected]">
3430
<autoRebuild>false</autoRebuild>
3531
<rebuildDisabled>false</rebuildDisabled>
@@ -48,70 +44,131 @@
4844
</triggers>
4945
<concurrentBuild>false</concurrentBuild>
5046
<builders>
51-
<javaposse.jobdsl.plugin.ExecuteDslScripts plugin="job-dsl@1.70">
47+
<javaposse.jobdsl.plugin.ExecuteDslScripts plugin="job-dsl@1.77">
5248
<scriptText>// Groovy script used to seed Jenkins with multi-branch pipeline jobs:
5349
// 1. Call GitLab API to get each git project in a given group
5450
// 2. Check if project is archived, if so skip it.
5551
// 3. Check if there is a Jenkinsfile (on master) in each of the found projects
5652
// 4. Generate a pipeline using the Jenkinsfile and add it to the queue on first creation
5753
// 5. Every 10 mins run again
5854

59-
def gitlabHost = System.getenv(&quot;GITLAB_HOST&quot;) ?: &quot;https://gitlab.apps.proj.example.com&quot;
60-
def gitlabToken = System.getenv(&quot;GITLAB_TOKEN&quot;) ?: &quot;mytoken123&quot;
61-
def projectName = System.getenv(&quot;GITLAB_GROUP_NAME&quot;) ?: &quot;rht-labs&quot;
62-
def projectsApi = new URL(&quot;${gitlabHost}/api/v4/groups/${projectName}/projects?per_page=100&quot;)
6355

56+
// GITLAB
57+
def gitlabHost = System.getenv("GITLAB_HOST") ?: "https://gitlab.apps.proj.example.com"
58+
def gitlabToken = System.getenv("GITLAB_TOKEN")
59+
def projectName = System.getenv("GITLAB_GROUP_NAME") ?: "rht-labs"
60+
def gitlabProjectsApi = new URL("${gitlabHost}/api/v4/groups/${projectName}/projects?per_page=100")
6461

65-
try {
66-
def projects = new groovy.json.JsonSlurper().parse(projectsApi.newReader(requestProperties: [&apos;PRIVATE-TOKEN&apos;: gitlabToken]))
6762

68-
projects.each {
69-
def project = &quot;${it.path}&quot;
70-
def gitPath = it.http_url_to_repo
63+
// GITHUB
64+
def githubHost = "https://api.github.com"
65+
// Token needed for rate limiting issues....
66+
def githubToken = System.getenv("GITHUB_TOKEN")
67+
def githubAccount = System.getenv("GITHUB_ACCOUNT")
68+
def githubOrg = System.getenv("GITHUB_ORG") ?: false
69+
//eg https://api.github.com/users/springdo/repos or
7170

72-
if (it.archived) {
73-
print &quot;skipping project ${project} because it has been archived\n\n&quot;
74-
return
75-
}
71+
def githubProjects = githubOrg ? new URL("${githubHost}/org/${githubAccount}/repos?per_page=100") : new URL("${githubHost}/users/${githubAccount}/repos?per_page=100")
7672

77-
try {
78-
def filesApi = new URL(&quot;${gitlabHost}/api/v4/projects/${it.id}/repository/files/Jenkinsfile?ref=master&quot;)
79-
def files = new groovy.json.JsonSlurper().parse(filesApi.newReader(requestProperties: [&apos;PRIVATE-TOKEN&apos;: gitlabToken]))
8073

81-
if (!jenkins.model.Jenkins.instance.getItemByFullName(project)) {
82-
print &quot;About to create ${project} for the first time, this will result in a triggering the build after this run to prepare the ${project} pipeline\n\n&quot;
83-
queue(project)
84-
}
74+
def createMultibranchPipelineJob(project, gitPath) {
75+
def buildNamespace = System.getenv("BUILD_NAMESPACE") ?: "labs-ci-cd"
76+
def buildGitAuthSecret = System.getenv("BUILD_GIT_AUTH_SECRET") ?: "git-auth"
8577

86-
// Build Jenkins multibranc jobs
87-
multibranchPipelineJob(project) {
88-
branchSources {
89-
git {
90-
remote(gitPath)
91-
credentialsId(&apos;labs-ci-cd-jenkins-git-password&apos;)
92-
}
93-
}
94-
triggers {
95-
computedFolderWebHookTrigger {
96-
// The token to match with webhook token.
97-
token(project)
98-
}
99-
}
100-
orphanedItemStrategy {
101-
discardOldItems {
102-
numToKeep(10)
103-
}
104-
}
78+
// Build Jenkins multibranc jobs
79+
multibranchPipelineJob(project) {
80+
branchSources {
81+
git {
82+
id("${project}")
83+
remote(gitPath)
84+
credentialsId("${buildNamespace}-${buildGitAuthSecret}")
85+
}
86+
}
87+
triggers {
88+
computedFolderWebHookTrigger {
89+
// The token to match with webhook token.
90+
token(project)
10591
}
10692
}
107-
catch(Exception e) {
108-
println e
109-
print &quot;skipping project ${project} because it has no Jenkinsfile\n\n&quot;
93+
orphanedItemStrategy {
94+
discardOldItems {
95+
// Set to 0 to autoprune jobs once branch is deleted
96+
numToKeep(0)
97+
}
11098
}
11199
}
112-
} catch(Exception e) {
113-
print &quot;\n\n Please make sure you have set GITLAB_HOST, GITLAB_TOKEN and GITLAB_GROUP_NAME in your deploy config for Jenkins \n\n\n&quot;
114-
throw e
100+
}
101+
102+
def addJobToQueue(project){
103+
if (!jenkins.model.Jenkins.instance.getItemByFullName(project)) {
104+
print "About to create ${project} for the first time, this will result in a triggering the build after this run to prepare the ${project} pipeline\n\n"
105+
queue(project)
106+
}
107+
}
108+
// if GITLAB* set ....
109+
if (gitlabToken) {
110+
try {
111+
def projects = new groovy.json.JsonSlurper().parse(gitlabProjectsApi.newReader(requestProperties: ['PRIVATE-TOKEN': gitlabToken]))
112+
113+
projects.each {
114+
def project = "${it.path}"
115+
def gitPath = it.http_url_to_repo
116+
117+
if (it.archived) {
118+
print "skipping project ${project} because it has been archived\n\n"
119+
return
120+
}
121+
122+
try {
123+
def filesApi = new URL("${gitlabHost}/api/v4/projects/${it.id}/repository/files/Jenkinsfile?ref=master")
124+
def files = new groovy.json.JsonSlurper().parse(filesApi.newReader(requestProperties: ['PRIVATE-TOKEN': gitlabToken]))
125+
126+
createMultibranchPipelineJob(project, gitPath)
127+
addJobToQueue(project)
128+
129+
}
130+
catch(Exception e) {
131+
println e
132+
print "skipping project ${project} because it has no Jenkinsfile\n\n"
133+
}
134+
}
135+
} catch(Exception e) {
136+
print "\n\n Please make sure you have set GITLAB_HOST, GITLAB_TOKEN and GITLAB_GROUP_NAME in your deploy config for Jenkins \n\n\n"
137+
throw e
138+
}
139+
} else if (githubAccount) {
140+
try {
141+
def projects = new groovy.json.JsonSlurper().parse(githubProjects.newReader(requestProperties: ['Authorization': "token ${githubToken}"]))
142+
143+
projects.each {
144+
def project = it.name
145+
def gitPath = it.clone_url
146+
147+
if (it.archived) {
148+
print "skipping project ${project} because it has been archived\n\n"
149+
return
150+
}
151+
152+
try {
153+
// https://api.github.com/repos/$ORG or $USER/name/contents/Jenkinsfile
154+
def filesApi = new URL("${githubHost}/repos/${githubAccount}/${project}/contents/Jenkinsfile")
155+
def files = new groovy.json.JsonSlurper().parse(filesApi.newReader(requestProperties: ['Authorization': "token ${githubToken}"]))
156+
157+
158+
createMultibranchPipelineJob(project, gitPath)
159+
addJobToQueue(project)
160+
}
161+
catch(Exception e) {
162+
println e
163+
print "skipping project ${project} because it has no Jenkinsfile\n\n"
164+
}
165+
}
166+
} catch(Exception e) {
167+
print "\n\n Oops! something went wrong..... Try setting the GITHUB_* Env Vars \n\n\n"
168+
throw e
169+
}
170+
} else {
171+
print "\n\n No tokens set in the Environment eg GITHUB* or GITLAB* so not sure what to do ..... &#129335;‍♂️ \n\n\n"
115172
}</scriptText>
116173
<usingScriptText>true</usingScriptText>
117174
<sandbox>false</sandbox>

‎configuration/jobs/seed-multibranch-pipelines/seed-multibranch-pipeline.groovy‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
// Groovy script used to seed Jenkins with multi-branch pipeline jobs:
2+
// 1. Call GitLab API to get each git project in a given group
3+
// 2. Check if project is archived, if so skip it.
4+
// 3. Check if there is a Jenkinsfile (on master) in each of the found projects
5+
// 4. Generate a pipeline using the Jenkinsfile and add it to the queue on first creation
6+
// 5. Every 10 mins run again
7+
8+
9+
// GITLAB
10+
def gitlabHost = System.getenv("GITLAB_HOST") ?: "https://gitlab.apps.proj.example.com"
11+
def gitlabToken = System.getenv("GITLAB_TOKEN")
12+
def projectName = System.getenv("GITLAB_GROUP_NAME") ?: "rht-labs"
13+
def gitlabProjectsApi = new URL("${gitlabHost}/api/v4/groups/${projectName}/projects?per_page=100")
14+
15+
16+
// GITHUB
17+
def githubHost = "https://api.github.com"
18+
// Token needed for rate limiting issues....
19+
def githubToken = System.getenv("GITHUB_TOKEN")
20+
def githubAccount = System.getenv("GITHUB_ACCOUNT")
21+
def githubOrg = System.getenv("GITHUB_ORG") ?: false
22+
//eg https://api.github.com/users/springdo/repos or
23+
24+
def githubProjects = githubOrg ? new URL("${githubHost}/org/${githubAccount}/repos?per_page=100") : new URL("${githubHost}/users/${githubAccount}/repos?per_page=100")
25+
26+
27+
def createMultibranchPipelineJob(project, gitPath) {
28+
def buildNamespace = System.getenv("BUILD_NAMESPACE") ?: "labs-ci-cd"
29+
def buildGitAuthSecret = System.getenv("BUILD_GIT_AUTH_SECRET") ?: "git-auth"
30+
31+
// Build Jenkins multibranc jobs
32+
multibranchPipelineJob(project) {
33+
branchSources {
34+
git {
35+
id("${project}")
36+
remote(gitPath)
37+
credentialsId("${buildNamespace}-${buildGitAuthSecret}")
38+
}
39+
}
40+
triggers {
41+
computedFolderWebHookTrigger {
42+
// The token to match with webhook token.
43+
token(project)
44+
}
45+
}
46+
orphanedItemStrategy {
47+
discardOldItems {
48+
// Set to 0 to autoprune jobs once branch is deleted
49+
numToKeep(0)
50+
}
51+
}
52+
}
53+
}
54+
55+
def addJobToQueue(project){
56+
if (!jenkins.model.Jenkins.instance.getItemByFullName(project)) {
57+
print "About to create ${project} for the first time, this will result in a triggering the build after this run to prepare the ${project} pipeline\n\n"
58+
queue(project)
59+
}
60+
}
61+
// if GITLAB* set ....
62+
if (gitlabToken) {
63+
try {
64+
def projects = new groovy.json.JsonSlurper().parse(gitlabProjectsApi.newReader(requestProperties: ['PRIVATE-TOKEN': gitlabToken]))
65+
66+
projects.each {
67+
def project = "${it.path}"
68+
def gitPath = it.http_url_to_repo
69+
70+
if (it.archived) {
71+
print "skipping project ${project} because it has been archived\n\n"
72+
return
73+
}
74+
75+
try {
76+
def filesApi = new URL("${gitlabHost}/api/v4/projects/${it.id}/repository/files/Jenkinsfile?ref=master")
77+
def files = new groovy.json.JsonSlurper().parse(filesApi.newReader(requestProperties: ['PRIVATE-TOKEN': gitlabToken]))
78+
79+
createMultibranchPipelineJob(project, gitPath)
80+
addJobToQueue(project)
81+
82+
}
83+
catch(Exception e) {
84+
println e
85+
print "skipping project ${project} because it has no Jenkinsfile\n\n"
86+
}
87+
}
88+
} catch(Exception e) {
89+
print "\n\n Please make sure you have set GITLAB_HOST, GITLAB_TOKEN and GITLAB_GROUP_NAME in your deploy config for Jenkins \n\n\n"
90+
throw e
91+
}
92+
} else if (githubAccount) {
93+
try {
94+
def projects = new groovy.json.JsonSlurper().parse(githubProjects.newReader(requestProperties: ['Authorization': "token ${githubToken}"]))
95+
96+
projects.each {
97+
def project = it.name
98+
def gitPath = it.clone_url
99+
100+
if (it.archived) {
101+
print "skipping project ${project} because it has been archived\n\n"
102+
return
103+
}
104+
105+
try {
106+
// https://api.github.com/repos/$ORG or $USER/name/contents/Jenkinsfile
107+
def filesApi = new URL("${githubHost}/repos/${githubAccount}/${project}/contents/Jenkinsfile")
108+
def files = new groovy.json.JsonSlurper().parse(filesApi.newReader(requestProperties: ['Authorization': "token ${githubToken}"]))
109+
110+
111+
createMultibranchPipelineJob(project, gitPath)
112+
addJobToQueue(project)
113+
}
114+
catch(Exception e) {
115+
println e
116+
print "skipping project ${project} because it has no Jenkinsfile\n\n"
117+
}
118+
}
119+
} catch(Exception e) {
120+
print "\n\n Oops! something went wrong..... Try setting the GITHUB_* Env Vars \n\n\n"
121+
throw e
122+
}
123+
} else {
124+
print "\n\n No tokens set in the Environment eg GITHUB* or GITLAB* so not sure what to do ..... 🤷‍♂️ \n\n\n"
125+
}

‎plugins.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ config-file-provider:3.6.3
5050
copy-to-slave:1.4.4
5151
credentials-binding:1.23
5252
credentials:2.3.7
53+
cucumber-reports:5.3.0
5354
cvs:2.16
5455
dashboard-view:2.12
5556
display-url-api:2.3.2

0 commit comments

Comments
 (0)