Skip to content

Commit ce94834

Browse files
jglickjglick
committed
New option SCMSourceRetriever.clone to avoid workspace/xxx@libs/ 
1 parent a1612bb commit ce94834

File tree

4 files changed

+202
-1
lines changed

4 files changed

+202
-1
lines changed

‎src/main/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever.java

Lines changed: 87 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,11 +60,14 @@
6060
import java.util.logging.Logger;
6161
import edu.umd.cs.findbugs.annotations.NonNull;
6262
import edu.umd.cs.findbugs.annotations.CheckForNull;
63+
import java.util.Set;
64+
import java.util.TreeSet;
6365
import jenkins.model.Jenkins;
6466
import jenkins.scm.api.SCMRevision;
6567
import jenkins.scm.api.SCMSource;
6668
import jenkins.scm.api.SCMSourceDescriptor;
6769
import java.util.regex.Pattern;
70+
import java.util.stream.Collectors;
6871
import org.jenkinsci.Symbol;
6972
import org.jenkinsci.plugins.structs.describable.CustomDescribableModel;
7073
import org.jenkinsci.plugins.structs.describable.UninstantiatedDescribable;
@@ -96,6 +99,8 @@ public class SCMSourceRetriever extends LibraryRetriever {
9699

97100
private final SCMSource scm;
98101

102+
private boolean clone;
103+
99104
/**
100105
* The path to the library inside of the SCM.
101106
*
@@ -117,6 +122,14 @@ public SCMSource getScm() {
117122
return scm;
118123
}
119124

125+
public boolean isClone() {
126+
return clone;
127+
}
128+
129+
@DataBoundSetter public void setClone(boolean clone) {
130+
this.clone = clone;
131+
}
132+
120133
public String getLibraryPath() {
121134
return libraryPath;
122135
}
@@ -134,7 +147,14 @@ public String getLibraryPath() {
134147
if (revision == null) {
135148
throw new AbortException("No version " + version + " found for library " + name);
136149
}
137-
doRetrieve(name, changelog, scm.build(revision.getHead(), revision), libraryPath, target, run, listener);
150+
if (clone) {
151+
if (changelog) {
152+
listener.getLogger().println("WARNING: ignoring request to compute changelog in clone mode");
153+
}
154+
doClone(scm.build(revision.getHead(), revision), libraryPath, target, run, listener);
155+
} else {
156+
doRetrieve(name, changelog, scm.build(revision.getHead(), revision), libraryPath, target, run, listener);
157+
}
138158
}
139159

140160
@Override public void retrieve(String name, String version, FilePath target, Run<?, ?> run, TaskListener listener) throws Exception {
@@ -221,6 +241,72 @@ private static String getFilePathSuffix() {
221241
return System.getProperty(WorkspaceList.class.getName(), "@");
222242
}
223243

244+
/**
245+
* Similar to {@link #doRetrieve} but used in {@link #clone} mode.
246+
*/
247+
private static void doClone(@NonNull SCM scm, String libraryPath, FilePath target, Run<?, ?> run, TaskListener listener) throws Exception {
248+
SCMStep delegate = new GenericSCMStep(scm);
249+
delegate.setPoll(false);
250+
delegate.setChangelog(false);
251+
if (libraryPath == null) {
252+
retrySCMOperation(listener, () -> {
253+
delegate.checkout(run, target, listener, Jenkins.get().createLauncher(listener));
254+
return null;
255+
});
256+
} else {
257+
if (PROHIBITED_DOUBLE_DOT.matcher(libraryPath).matches()) {
258+
throw new AbortException("Library path may not contain '..'");
259+
}
260+
FilePath root = target.child("root");
261+
retrySCMOperation(listener, () -> {
262+
delegate.checkout(run, root, listener, Jenkins.get().createLauncher(listener));
263+
return null;
264+
});
265+
FilePath subdir = root.child(libraryPath);
266+
if (!subdir.isDirectory()) {
267+
throw new AbortException("Did not find " + libraryPath + " in checkout");
268+
}
269+
for (String content : List.of("src", "vars", "resources")) {
270+
FilePath contentDir = subdir.child(content);
271+
if (contentDir.isDirectory()) {
272+
listener.getLogger().println("Moving " + content + " to top level");
273+
contentDir.renameTo(target.child(content));
274+
}
275+
}
276+
// root itself will be deleted below
277+
}
278+
Set<String> deleted = new TreeSet<>();
279+
if (!INCLUDE_SRC_TEST_IN_LIBRARIES) {
280+
FilePath srcTest = target.child("src/test");
281+
if (srcTest.isDirectory()) {
282+
listener.getLogger().println("Excluding src/test/ from checkout of " + scm.getKey() + " so that library test code cannot be accessed by Pipelines.");
283+
listener.getLogger().println("To remove this log message, move the test code outside of src/. To restore the previous behavior that allowed access to files in src/test/, pass -D" + SCMSourceRetriever.class.getName() + ".INCLUDE_SRC_TEST_IN_LIBRARIES=true to the java command used to start Jenkins.");
284+
srcTest.deleteRecursive();
285+
deleted.add("src/test");
286+
}
287+
}
288+
for (FilePath child : target.list()) {
289+
String name = child.getName();
290+
switch (name) {
291+
case "src":
292+
// TODO delete everything that is not *.groovy
293+
break;
294+
case "vars":
295+
// TODO delete everything that is not *.groovy or *.txt, incl. subdirs
296+
break;
297+
case "resources":
298+
// OK, leave it all
299+
break;
300+
default:
301+
deleted.add(name);
302+
child.deleteRecursive();
303+
}
304+
}
305+
if (!deleted.isEmpty()) {
306+
listener.getLogger().println("Deleted " + deleted.stream().collect(Collectors.joining(", ")));
307+
}
308+
}
309+
224310
@Override public FormValidation validateVersion(String name, String version, Item context) {
225311
StringWriter w = new StringWriter();
226312
try {

‎src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/config.jelly

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@ THE SOFTWARE.
2727
<j:jelly xmlns:j="jelly:core" xmlns:f="/lib/form">
2828
<f:description>${%blurb}</f:description>
2929
<f:dropdownDescriptorSelector field="scm" descriptors="${descriptor.SCMDescriptors}" title="${%Source Code Management}"/>
30+
<f:entry field="clone" title="${%Fresh clone per build}">
31+
<f:checkbox/>
32+
</f:entry>
3033
<f:entry field="libraryPath" title="${%libraryPath}">
3134
<f:textbox checkMethod="post"/>
3235
</f:entry>

‎src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/help-clone.html

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
<div>
2+
If checked, every build performs a fresh clone of the SCM rather than locking and updating a common copy.
3+
No changelog will be computed.
4+
For Git, you are advised to select <b>Advanced clone behaviors » Shallow clone</b> to make the clone much faster.
5+
You may still enable <b>Cache fetched versions on controller for quick retrieval</b> if you prefer.
6+
</div>

‎src/test/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetrieverTest.java

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@
6060
import org.jenkinsci.plugins.workflow.job.WorkflowRun;
6161

6262
import static hudson.ExtensionList.lookupSingleton;
63+
import hudson.plugins.git.extensions.impl.CloneOption;
64+
import jenkins.plugins.git.traits.CloneOptionTrait;
65+
import jenkins.scm.api.trait.SCMSourceTrait;
6366
import static org.hamcrest.Matchers.contains;
6467
import static org.hamcrest.Matchers.containsInAnyOrder;
6568
import static org.hamcrest.Matchers.instanceOf;
@@ -81,13 +84,15 @@
8184
import static org.hamcrest.Matchers.nullValue;
8285
import static org.jenkinsci.plugins.workflow.libs.SCMSourceRetriever.PROHIBITED_DOUBLE_DOT;
8386
import static org.junit.Assume.assumeFalse;
87+
import org.jvnet.hudson.test.FlagRule;
8488

8589
public class SCMSourceRetrieverTest {
8690

8791
@ClassRule public static BuildWatcher buildWatcher = new BuildWatcher();
8892
@Rule public JenkinsRule r = new JenkinsRule();
8993
@Rule public GitSampleRepoRule sampleRepo = new GitSampleRepoRule();
9094
@Rule public SubversionSampleRepoRule sampleRepoSvn = new SubversionSampleRepoRule();
95+
@Rule public FlagRule<Boolean> includeSrcTest = new FlagRule<>(() -> SCMSourceRetriever.INCLUDE_SRC_TEST_IN_LIBRARIES, v -> SCMSourceRetriever.INCLUDE_SRC_TEST_IN_LIBRARIES = v);
9196

9297
@Issue("JENKINS-40408")
9398
@Test public void lease() throws Exception {
@@ -366,6 +371,107 @@ public static class BasicSCMSource extends SCMSource {
366371
assertFalse(ws.exists());
367372
}
368373

374+
@Test public void cloneMode() throws Exception {
375+
sampleRepo.init();
376+
sampleRepo.write("vars/myecho.groovy", "def call() {echo 'something special'}");
377+
sampleRepo.write("README.md", "Summary");
378+
sampleRepo.git("rm", "file");
379+
sampleRepo.git("add", ".");
380+
sampleRepo.git("commit", "--message=init");
381+
GitSCMSource src = new GitSCMSource(sampleRepo.toString());
382+
src.setTraits(List.<SCMSourceTrait>of(new CloneOptionTrait(new CloneOption(true, null, null))));
383+
SCMSourceRetriever scm = new SCMSourceRetriever(src);
384+
LibraryConfiguration lc = new LibraryConfiguration("echoing", scm);
385+
lc.setIncludeInChangesets(false);
386+
scm.setClone(true);
387+
GlobalLibraries.get().setLibraries(Collections.singletonList(lc));
388+
WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p");
389+
p.setDefinition(new CpsFlowDefinition("@Library('echoing@master') import myecho; myecho()", true));
390+
WorkflowRun b = r.buildAndAssertSuccess(p);
391+
assertFalse(r.jenkins.getWorkspaceFor(p).withSuffix("@libs").isDirectory());
392+
r.assertLogContains("something special", b);
393+
r.assertLogContains("Deleted .git, README.md", b);
394+
r.assertLogContains("Using shallow clone with depth 1", b);
395+
}
396+
397+
@Test public void cloneModeLibraryPath() throws Exception {
398+
sampleRepo.init();
399+
sampleRepo.write("sub/path/vars/myecho.groovy", "def call() {echo 'something special'}");
400+
sampleRepo.git("add", "sub");
401+
sampleRepo.git("commit", "--message=init");
402+
SCMSourceRetriever scm = new SCMSourceRetriever(new GitSCMSource(sampleRepo.toString()));
403+
LibraryConfiguration lc = new LibraryConfiguration("root_sub_path", scm);
404+
lc.setIncludeInChangesets(false);
405+
scm.setLibraryPath("sub/path/");
406+
scm.setClone(true);
407+
GlobalLibraries.get().setLibraries(Collections.singletonList(lc));
408+
WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p");
409+
p.setDefinition(new CpsFlowDefinition("@Library('root_sub_path@master') import myecho; myecho()", true));
410+
WorkflowRun b = r.buildAndAssertSuccess(p);
411+
r.assertLogContains("something special", b);
412+
r.assertLogContains("Moving vars to top level", b);
413+
r.assertLogContains("Deleted root", b);
414+
}
415+
416+
@Test public void cloneModeLibraryPathSecurity() throws Exception {
417+
sampleRepo.init();
418+
sampleRepo.write("sub/path/vars/myecho.groovy", "def call() {echo 'something special'}");
419+
sampleRepo.git("add", "sub");
420+
sampleRepo.git("commit", "--message=init");
421+
SCMSourceRetriever scm = new SCMSourceRetriever(new GitSCMSource(sampleRepo.toString()));
422+
LibraryConfiguration lc = new LibraryConfiguration("root_sub_path", scm);
423+
lc.setIncludeInChangesets(false);
424+
scm.setLibraryPath("sub/path/../../../jenkins_home/foo");
425+
scm.setClone(true);
426+
GlobalLibraries.get().setLibraries(Collections.singletonList(lc));
427+
WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p");
428+
p.setDefinition(new CpsFlowDefinition("@Library('root_sub_path@master') import myecho; myecho()", true));
429+
WorkflowRun b = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0));
430+
r.assertLogContains("Library path may not contain '..'", b);
431+
}
432+
433+
@Test public void cloneModeExcludeSrcTest() throws Exception {
434+
sampleRepo.init();
435+
sampleRepo.write("vars/myecho.groovy", "def call() {echo 'something special'}");
436+
sampleRepo.write("src/test/X.groovy", "// irrelevant");
437+
sampleRepo.write("README.md", "Summary");
438+
sampleRepo.git("add", ".");
439+
sampleRepo.git("commit", "--message=init");
440+
SCMSourceRetriever scm = new SCMSourceRetriever(new GitSCMSource(sampleRepo.toString()));
441+
LibraryConfiguration lc = new LibraryConfiguration("echoing", scm);
442+
lc.setIncludeInChangesets(false);
443+
scm.setClone(true);
444+
GlobalLibraries.get().setLibraries(Collections.singletonList(lc));
445+
WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p");
446+
p.setDefinition(new CpsFlowDefinition("@Library('echoing@master') import myecho; myecho()", true));
447+
SCMSourceRetriever.INCLUDE_SRC_TEST_IN_LIBRARIES = false;
448+
WorkflowRun b = r.buildAndAssertSuccess(p);
449+
assertFalse(r.jenkins.getWorkspaceFor(p).withSuffix("@libs").isDirectory());
450+
r.assertLogContains("something special", b);
451+
r.assertLogContains("Excluding src/test/ from checkout", b);
452+
}
453+
454+
@Test public void cloneModeIncludeSrcTest() throws Exception {
455+
sampleRepo.init();
456+
sampleRepo.write("vars/myecho.groovy", "def call() {echo(/got ${new test.X().m()}/)}");
457+
sampleRepo.write("src/test/X.groovy", "package test; class X {def m() {'something special'}}");
458+
sampleRepo.write("README.md", "Summary");
459+
sampleRepo.git("add", ".");
460+
sampleRepo.git("commit", "--message=init");
461+
SCMSourceRetriever scm = new SCMSourceRetriever(new GitSCMSource(sampleRepo.toString()));
462+
LibraryConfiguration lc = new LibraryConfiguration("echoing", scm);
463+
lc.setIncludeInChangesets(false);
464+
scm.setClone(true);
465+
GlobalLibraries.get().setLibraries(Collections.singletonList(lc));
466+
WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p");
467+
p.setDefinition(new CpsFlowDefinition("@Library('echoing@master') import myecho; myecho()", true));
468+
SCMSourceRetriever.INCLUDE_SRC_TEST_IN_LIBRARIES = true;
469+
WorkflowRun b = r.buildAndAssertSuccess(p);
470+
assertFalse(r.jenkins.getWorkspaceFor(p).withSuffix("@libs").isDirectory());
471+
r.assertLogContains("got something special", b);
472+
r.assertLogNotContains("Excluding src/test/ from checkout", b);
473+
}
474+
369475
@Issue("SECURITY-2441")
370476
@Test public void libraryNamesAreNotUsedAsCheckoutDirectories() throws Exception {
371477
sampleRepo.init();

0 commit comments

Comments
 (0)