From ce94834b03f96b654f9616364a9eaecc38042fc9 Mon Sep 17 00:00:00 2001 From: Jesse Glick Date: Thu, 2 Mar 2023 14:35:05 -0500 Subject: [PATCH 1/4] New option `SCMSourceRetriever.clone` to avoid `workspace/xxx@libs/` --- .../workflow/libs/SCMSourceRetriever.java | 88 ++++++++++++++- .../libs/SCMSourceRetriever/config.jelly | 3 + .../libs/SCMSourceRetriever/help-clone.html | 6 + .../workflow/libs/SCMSourceRetrieverTest.java | 106 ++++++++++++++++++ 4 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/help-clone.html diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever.java index 9732508b..8b42c8f0 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever.java @@ -60,11 +60,14 @@ import java.util.logging.Logger; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.CheckForNull; +import java.util.Set; +import java.util.TreeSet; import jenkins.model.Jenkins; import jenkins.scm.api.SCMRevision; import jenkins.scm.api.SCMSource; import jenkins.scm.api.SCMSourceDescriptor; import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.jenkinsci.Symbol; import org.jenkinsci.plugins.structs.describable.CustomDescribableModel; import org.jenkinsci.plugins.structs.describable.UninstantiatedDescribable; @@ -96,6 +99,8 @@ public class SCMSourceRetriever extends LibraryRetriever { private final SCMSource scm; + private boolean clone; + /** * The path to the library inside of the SCM. * @@ -117,6 +122,14 @@ public SCMSource getScm() { return scm; } + public boolean isClone() { + return clone; + } + + @DataBoundSetter public void setClone(boolean clone) { + this.clone = clone; + } + public String getLibraryPath() { return libraryPath; } @@ -134,7 +147,14 @@ public String getLibraryPath() { if (revision == null) { throw new AbortException("No version " + version + " found for library " + name); } - doRetrieve(name, changelog, scm.build(revision.getHead(), revision), libraryPath, target, run, listener); + if (clone) { + if (changelog) { + listener.getLogger().println("WARNING: ignoring request to compute changelog in clone mode"); + } + doClone(scm.build(revision.getHead(), revision), libraryPath, target, run, listener); + } else { + doRetrieve(name, changelog, scm.build(revision.getHead(), revision), libraryPath, target, run, listener); + } } @Override public void retrieve(String name, String version, FilePath target, Run run, TaskListener listener) throws Exception { @@ -221,6 +241,72 @@ private static String getFilePathSuffix() { return System.getProperty(WorkspaceList.class.getName(), "@"); } + /** + * Similar to {@link #doRetrieve} but used in {@link #clone} mode. + */ + private static void doClone(@NonNull SCM scm, String libraryPath, FilePath target, Run run, TaskListener listener) throws Exception { + SCMStep delegate = new GenericSCMStep(scm); + delegate.setPoll(false); + delegate.setChangelog(false); + if (libraryPath == null) { + retrySCMOperation(listener, () -> { + delegate.checkout(run, target, listener, Jenkins.get().createLauncher(listener)); + return null; + }); + } else { + if (PROHIBITED_DOUBLE_DOT.matcher(libraryPath).matches()) { + throw new AbortException("Library path may not contain '..'"); + } + FilePath root = target.child("root"); + retrySCMOperation(listener, () -> { + delegate.checkout(run, root, listener, Jenkins.get().createLauncher(listener)); + return null; + }); + FilePath subdir = root.child(libraryPath); + if (!subdir.isDirectory()) { + throw new AbortException("Did not find " + libraryPath + " in checkout"); + } + for (String content : List.of("src", "vars", "resources")) { + FilePath contentDir = subdir.child(content); + if (contentDir.isDirectory()) { + listener.getLogger().println("Moving " + content + " to top level"); + contentDir.renameTo(target.child(content)); + } + } + // root itself will be deleted below + } + Set deleted = new TreeSet<>(); + if (!INCLUDE_SRC_TEST_IN_LIBRARIES) { + FilePath srcTest = target.child("src/test"); + if (srcTest.isDirectory()) { + listener.getLogger().println("Excluding src/test/ from checkout of " + scm.getKey() + " so that library test code cannot be accessed by Pipelines."); + listener.getLogger().println("To remove this log message, move the test code outside of src/. To restore the previous behavior that allowed access to files in src/test/, pass -D" + SCMSourceRetriever.class.getName() + ".INCLUDE_SRC_TEST_IN_LIBRARIES=true to the java command used to start Jenkins."); + srcTest.deleteRecursive(); + deleted.add("src/test"); + } + } + for (FilePath child : target.list()) { + String name = child.getName(); + switch (name) { + case "src": + // TODO delete everything that is not *.groovy + break; + case "vars": + // TODO delete everything that is not *.groovy or *.txt, incl. subdirs + break; + case "resources": + // OK, leave it all + break; + default: + deleted.add(name); + child.deleteRecursive(); + } + } + if (!deleted.isEmpty()) { + listener.getLogger().println("Deleted " + deleted.stream().collect(Collectors.joining(", "))); + } + } + @Override public FormValidation validateVersion(String name, String version, Item context) { StringWriter w = new StringWriter(); try { diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/config.jelly b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/config.jelly index d7489eb1..331ea146 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/config.jelly +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/config.jelly @@ -27,6 +27,9 @@ THE SOFTWARE. ${%blurb} + + + diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/help-clone.html b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/help-clone.html new file mode 100644 index 00000000..2f375430 --- /dev/null +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/help-clone.html @@ -0,0 +1,6 @@ +
+ If checked, every build performs a fresh clone of the SCM rather than locking and updating a common copy. + No changelog will be computed. + For Git, you are advised to select Advanced clone behaviors » Shallow clone to make the clone much faster. + You may still enable Cache fetched versions on controller for quick retrieval if you prefer. +
diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetrieverTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetrieverTest.java index e1b1553a..91806f00 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetrieverTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetrieverTest.java @@ -60,6 +60,9 @@ import org.jenkinsci.plugins.workflow.job.WorkflowRun; import static hudson.ExtensionList.lookupSingleton; +import hudson.plugins.git.extensions.impl.CloneOption; +import jenkins.plugins.git.traits.CloneOptionTrait; +import jenkins.scm.api.trait.SCMSourceTrait; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.instanceOf; @@ -81,6 +84,7 @@ import static org.hamcrest.Matchers.nullValue; import static org.jenkinsci.plugins.workflow.libs.SCMSourceRetriever.PROHIBITED_DOUBLE_DOT; import static org.junit.Assume.assumeFalse; +import org.jvnet.hudson.test.FlagRule; public class SCMSourceRetrieverTest { @@ -88,6 +92,7 @@ public class SCMSourceRetrieverTest { @Rule public JenkinsRule r = new JenkinsRule(); @Rule public GitSampleRepoRule sampleRepo = new GitSampleRepoRule(); @Rule public SubversionSampleRepoRule sampleRepoSvn = new SubversionSampleRepoRule(); + @Rule public FlagRule includeSrcTest = new FlagRule<>(() -> SCMSourceRetriever.INCLUDE_SRC_TEST_IN_LIBRARIES, v -> SCMSourceRetriever.INCLUDE_SRC_TEST_IN_LIBRARIES = v); @Issue("JENKINS-40408") @Test public void lease() throws Exception { @@ -366,6 +371,107 @@ public static class BasicSCMSource extends SCMSource { assertFalse(ws.exists()); } + @Test public void cloneMode() throws Exception { + sampleRepo.init(); + sampleRepo.write("vars/myecho.groovy", "def call() {echo 'something special'}"); + sampleRepo.write("README.md", "Summary"); + sampleRepo.git("rm", "file"); + sampleRepo.git("add", "."); + sampleRepo.git("commit", "--message=init"); + GitSCMSource src = new GitSCMSource(sampleRepo.toString()); + src.setTraits(List.of(new CloneOptionTrait(new CloneOption(true, null, null)))); + SCMSourceRetriever scm = new SCMSourceRetriever(src); + LibraryConfiguration lc = new LibraryConfiguration("echoing", scm); + lc.setIncludeInChangesets(false); + scm.setClone(true); + GlobalLibraries.get().setLibraries(Collections.singletonList(lc)); + WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); + p.setDefinition(new CpsFlowDefinition("@Library('echoing@master') import myecho; myecho()", true)); + WorkflowRun b = r.buildAndAssertSuccess(p); + assertFalse(r.jenkins.getWorkspaceFor(p).withSuffix("@libs").isDirectory()); + r.assertLogContains("something special", b); + r.assertLogContains("Deleted .git, README.md", b); + r.assertLogContains("Using shallow clone with depth 1", b); + } + + @Test public void cloneModeLibraryPath() throws Exception { + sampleRepo.init(); + sampleRepo.write("sub/path/vars/myecho.groovy", "def call() {echo 'something special'}"); + sampleRepo.git("add", "sub"); + sampleRepo.git("commit", "--message=init"); + SCMSourceRetriever scm = new SCMSourceRetriever(new GitSCMSource(sampleRepo.toString())); + LibraryConfiguration lc = new LibraryConfiguration("root_sub_path", scm); + lc.setIncludeInChangesets(false); + scm.setLibraryPath("sub/path/"); + scm.setClone(true); + GlobalLibraries.get().setLibraries(Collections.singletonList(lc)); + WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); + p.setDefinition(new CpsFlowDefinition("@Library('root_sub_path@master') import myecho; myecho()", true)); + WorkflowRun b = r.buildAndAssertSuccess(p); + r.assertLogContains("something special", b); + r.assertLogContains("Moving vars to top level", b); + r.assertLogContains("Deleted root", b); + } + + @Test public void cloneModeLibraryPathSecurity() throws Exception { + sampleRepo.init(); + sampleRepo.write("sub/path/vars/myecho.groovy", "def call() {echo 'something special'}"); + sampleRepo.git("add", "sub"); + sampleRepo.git("commit", "--message=init"); + SCMSourceRetriever scm = new SCMSourceRetriever(new GitSCMSource(sampleRepo.toString())); + LibraryConfiguration lc = new LibraryConfiguration("root_sub_path", scm); + lc.setIncludeInChangesets(false); + scm.setLibraryPath("sub/path/../../../jenkins_home/foo"); + scm.setClone(true); + GlobalLibraries.get().setLibraries(Collections.singletonList(lc)); + WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); + p.setDefinition(new CpsFlowDefinition("@Library('root_sub_path@master') import myecho; myecho()", true)); + WorkflowRun b = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); + r.assertLogContains("Library path may not contain '..'", b); + } + + @Test public void cloneModeExcludeSrcTest() throws Exception { + sampleRepo.init(); + sampleRepo.write("vars/myecho.groovy", "def call() {echo 'something special'}"); + sampleRepo.write("src/test/X.groovy", "// irrelevant"); + sampleRepo.write("README.md", "Summary"); + sampleRepo.git("add", "."); + sampleRepo.git("commit", "--message=init"); + SCMSourceRetriever scm = new SCMSourceRetriever(new GitSCMSource(sampleRepo.toString())); + LibraryConfiguration lc = new LibraryConfiguration("echoing", scm); + lc.setIncludeInChangesets(false); + scm.setClone(true); + GlobalLibraries.get().setLibraries(Collections.singletonList(lc)); + WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); + p.setDefinition(new CpsFlowDefinition("@Library('echoing@master') import myecho; myecho()", true)); + SCMSourceRetriever.INCLUDE_SRC_TEST_IN_LIBRARIES = false; + WorkflowRun b = r.buildAndAssertSuccess(p); + assertFalse(r.jenkins.getWorkspaceFor(p).withSuffix("@libs").isDirectory()); + r.assertLogContains("something special", b); + r.assertLogContains("Excluding src/test/ from checkout", b); + } + + @Test public void cloneModeIncludeSrcTest() throws Exception { + sampleRepo.init(); + sampleRepo.write("vars/myecho.groovy", "def call() {echo(/got ${new test.X().m()}/)}"); + sampleRepo.write("src/test/X.groovy", "package test; class X {def m() {'something special'}}"); + sampleRepo.write("README.md", "Summary"); + sampleRepo.git("add", "."); + sampleRepo.git("commit", "--message=init"); + SCMSourceRetriever scm = new SCMSourceRetriever(new GitSCMSource(sampleRepo.toString())); + LibraryConfiguration lc = new LibraryConfiguration("echoing", scm); + lc.setIncludeInChangesets(false); + scm.setClone(true); + GlobalLibraries.get().setLibraries(Collections.singletonList(lc)); + WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); + p.setDefinition(new CpsFlowDefinition("@Library('echoing@master') import myecho; myecho()", true)); + SCMSourceRetriever.INCLUDE_SRC_TEST_IN_LIBRARIES = true; + WorkflowRun b = r.buildAndAssertSuccess(p); + assertFalse(r.jenkins.getWorkspaceFor(p).withSuffix("@libs").isDirectory()); + r.assertLogContains("got something special", b); + r.assertLogNotContains("Excluding src/test/ from checkout", b); + } + @Issue("SECURITY-2441") @Test public void libraryNamesAreNotUsedAsCheckoutDirectories() throws Exception { sampleRepo.init(); From 76ad53e052aa7b9a7d0101b23e83a8ed5ab43feb Mon Sep 17 00:00:00 2001 From: Jesse Glick Date: Fri, 24 Mar 2023 09:11:03 -0400 Subject: [PATCH 2/4] Factor out `SCMBasedRetriever` roughly backporting https://github.com/jenkinsci/pipeline-groovy-lib-plugin/pull/57/files/840cd6004501d8d9e52317288ebbbb86fddbd7a4..444bde7967e0fa5ad9242bb1c2e08a073b49f21e + https://github.com/jenkinsci/pipeline-groovy-lib-plugin/pull/57/commits/e5e95d0d2f92f5091f1646915f82a88e7e2a5590 --- .../workflow/libs/SCMBasedRetriever.java | 336 ++++++++++++++++++ .../plugins/workflow/libs/SCMRetriever.java | 40 +-- .../workflow/libs/SCMSourceRetriever.java | 303 +--------------- .../help-clone.html | 0 .../help-libraryPath.html | 0 .../libs/SCMBasedRetriever/options.jelly | 34 ++ .../libs/SCMBasedRetriever/options.properties | 23 ++ .../workflow/libs/SCMRetriever/config.jelly | 6 +- .../libs/SCMRetriever/config.properties | 1 - .../libs/SCMSourceRetriever/config.jelly | 9 +- .../libs/SCMSourceRetriever/config.properties | 1 - .../SCMSourceRetriever/help-libraryPath.html | 5 - .../workflow/libs/SCMSourceRetrieverTest.java | 8 +- 13 files changed, 408 insertions(+), 358 deletions(-) create mode 100644 src/main/java/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever.java rename src/main/resources/org/jenkinsci/plugins/workflow/libs/{SCMSourceRetriever => SCMBasedRetriever}/help-clone.html (100%) rename src/main/resources/org/jenkinsci/plugins/workflow/libs/{SCMRetriever => SCMBasedRetriever}/help-libraryPath.html (100%) create mode 100644 src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/options.jelly create mode 100644 src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/options.properties delete mode 100644 src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/help-libraryPath.html diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever.java new file mode 100644 index 00000000..cf6c75d1 --- /dev/null +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever.java @@ -0,0 +1,336 @@ +/* + * The MIT License + * + * Copyright 2023 CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package org.jenkinsci.plugins.workflow.libs; + +import edu.umd.cs.findbugs.annotations.CheckForNull; +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import hudson.AbortException; +import hudson.Extension; +import hudson.FilePath; +import hudson.Functions; +import hudson.Util; +import hudson.model.Computer; +import hudson.model.Item; +import hudson.model.Job; +import hudson.model.Node; +import hudson.model.Run; +import hudson.model.TaskListener; +import hudson.model.TopLevelItem; +import hudson.model.listeners.ItemListener; +import hudson.scm.SCM; +import hudson.slaves.WorkspaceList; +import hudson.util.FormValidation; +import java.io.File; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.Callable; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import jenkins.model.Jenkins; +import org.jenkinsci.plugins.workflow.steps.scm.GenericSCMStep; +import org.jenkinsci.plugins.workflow.steps.scm.SCMStep; +import org.kohsuke.accmod.Restricted; +import org.kohsuke.accmod.restrictions.DoNotUse; +import org.kohsuke.stapler.DataBoundSetter; +import org.kohsuke.stapler.QueryParameter; +import org.kohsuke.stapler.verb.POST; + +/** + * Functionality common to {@link SCMSourceRetriever} and {@link SCMRetriever}. + */ +public abstract class SCMBasedRetriever extends LibraryRetriever { + + private static final Logger LOGGER = Logger.getLogger(SCMBasedRetriever.class.getName()); + + @SuppressFBWarnings(value = "MS_SHOULD_BE_FINAL", justification = "Non-final for write access via the Script Console") + public static boolean INCLUDE_SRC_TEST_IN_LIBRARIES = Boolean.getBoolean(SCMSourceRetriever.class.getName() + ".INCLUDE_SRC_TEST_IN_LIBRARIES"); + + /** + * Matches ".." in positions where it would be treated as the parent directory. + * + *

Used to prevent {@link #libraryPath} from being used for directory traversal. + */ + static final Pattern PROHIBITED_DOUBLE_DOT = Pattern.compile("(^|.*[\\\\/])\\.\\.($|[\\\\/].*)"); + + private boolean clone; + + /** + * The path to the library inside of the SCM. + * + * {@code null} is the default and means that the library is in the root of the repository. Otherwise, the value is + * considered to be a relative path inside of the repository and always ends in a forward slash + * + * @see #setLibraryPath + */ + private @CheckForNull String libraryPath; + + public boolean isClone() { + return clone; + } + + @DataBoundSetter public void setClone(boolean clone) { + this.clone = clone; + } + + public String getLibraryPath() { + return libraryPath; + } + + @DataBoundSetter public void setLibraryPath(String libraryPath) { + libraryPath = Util.fixEmptyAndTrim(libraryPath); + if (libraryPath != null && !libraryPath.endsWith("/")) { + libraryPath += '/'; + } + this.libraryPath = libraryPath; + } + + protected final void doRetrieve(String name, boolean changelog, @NonNull SCM scm, FilePath target, Run run, TaskListener listener) throws Exception { + if (libraryPath != null && PROHIBITED_DOUBLE_DOT.matcher(libraryPath).matches()) { + throw new AbortException("Library path may not contain '..'"); + } + if (clone && changelog) { + listener.getLogger().println("WARNING: ignoring request to compute changelog in clone mode"); + changelog = false; + } + // Adapted from CpsScmFlowDefinition: + SCMStep delegate = new GenericSCMStep(scm); + delegate.setPoll(false); // TODO we have no API for determining if a given SCMHead is branch-like or tag-like; would we want to turn on polling if the former? + delegate.setChangelog(changelog); + Node node = Jenkins.get(); + if (clone) { + if (libraryPath == null) { + retrySCMOperation(listener, () -> { + delegate.checkout(run, target, listener, Jenkins.get().createLauncher(listener)); + WorkspaceList.tempDir(target).deleteRecursive(); + return null; + }); + } else { + FilePath root = target.child("root"); + retrySCMOperation(listener, () -> { + delegate.checkout(run, root, listener, Jenkins.get().createLauncher(listener)); + WorkspaceList.tempDir(root).deleteRecursive(); + return null; + }); + FilePath subdir = root.child(libraryPath); + if (!subdir.isDirectory()) { + throw new AbortException("Did not find " + libraryPath + " in checkout"); + } + for (String content : List.of("src", "vars", "resources")) { + FilePath contentDir = subdir.child(content); + if (contentDir.isDirectory()) { + listener.getLogger().println("Moving " + content + " to top level"); + contentDir.renameTo(target.child(content)); + } + } + // root itself will be deleted below + } + Set deleted = new TreeSet<>(); + if (!INCLUDE_SRC_TEST_IN_LIBRARIES) { + FilePath srcTest = target.child("src/test"); + if (srcTest.isDirectory()) { + listener.getLogger().println("Excluding src/test/ from checkout of " + scm.getKey() + " so that library test code cannot be accessed by Pipelines."); + listener.getLogger().println("To remove this log message, move the test code outside of src/. To restore the previous behavior that allowed access to files in src/test/, pass -D" + SCMSourceRetriever.class.getName() + ".INCLUDE_SRC_TEST_IN_LIBRARIES=true to the java command used to start Jenkins."); + srcTest.deleteRecursive(); + deleted.add("src/test"); + } + } + for (FilePath child : target.list()) { + String subdir = child.getName(); + switch (subdir) { + case "src": + // TODO delete everything that is not *.groovy + break; + case "vars": + // TODO delete everything that is not *.groovy or *.txt, incl. subdirs + break; + case "resources": + // OK, leave it all + break; + default: + deleted.add(subdir); + child.deleteRecursive(); + } + } + if (!deleted.isEmpty()) { + listener.getLogger().println("Deleted " + deleted.stream().collect(Collectors.joining(", "))); + } + } else { // !clone + FilePath dir; + if (run.getParent() instanceof TopLevelItem) { + FilePath baseWorkspace = node.getWorkspaceFor((TopLevelItem) run.getParent()); + if (baseWorkspace == null) { + throw new IOException(node.getDisplayName() + " may be offline"); + } + String checkoutDirName = LibraryRecord.directoryNameFor(scm.getKey()); + dir = baseWorkspace.withSuffix(getFilePathSuffix() + "libs").child(checkoutDirName); + } else { // should not happen, but just in case: + throw new AbortException("Cannot check out in non-top-level build"); + } + Computer computer = node.toComputer(); + if (computer == null) { + throw new IOException(node.getDisplayName() + " may be offline"); + } + try (WorkspaceList.Lease lease = computer.getWorkspaceList().allocate(dir)) { + // Write the SCM key to a file as a debugging aid. + lease.path.withSuffix("-scm-key.txt").write(scm.getKey(), "UTF-8"); + retrySCMOperation(listener, () -> { + delegate.checkout(run, lease.path, listener, node.createLauncher(listener)); + return null; + }); + if (libraryPath == null) { + libraryPath = "."; + } + String excludes = INCLUDE_SRC_TEST_IN_LIBRARIES ? null : "src/test/"; + if (lease.path.child(libraryPath).child("src/test").exists()) { + listener.getLogger().println("Excluding src/test/ from checkout of " + scm.getKey() + " so that library test code cannot be accessed by Pipelines."); + listener.getLogger().println("To remove this log message, move the test code outside of src/. To restore the previous behavior that allowed access to files in src/test/, pass -D" + SCMSourceRetriever.class.getName() + ".INCLUDE_SRC_TEST_IN_LIBRARIES=true to the java command used to start Jenkins."); + } + // Cannot add WorkspaceActionImpl to private CpsFlowExecution.flowStartNodeActions; do we care? + // Copy sources with relevant files from the checkout: + lease.path.child(libraryPath).copyRecursiveTo("src/**/*.groovy,vars/*.groovy,vars/*.txt,resources/", excludes, target); + } + } + } + + protected static T retrySCMOperation(TaskListener listener, Callable task) throws Exception{ + T ret = null; + for (int retryCount = Jenkins.get().getScmCheckoutRetryCount(); retryCount >= 0; retryCount--) { + try { + ret = task.call(); + break; + } + catch (AbortException e) { + // abort exception might have a null message. + // If so, just skip echoing it. + if (e.getMessage() != null) { + listener.error(e.getMessage()); + } + } + catch (InterruptedIOException e) { + throw e; + } + catch (Exception e) { + // checkout error not yet reported + Functions.printStackTrace(e, listener.error("Checkout failed")); + } + + if (retryCount == 0) // all attempts failed + throw new AbortException("Maximum checkout retry attempts reached, aborting"); + + listener.getLogger().println("Retrying after 10 seconds"); + Thread.sleep(10000); + } + return ret; + } + + // TODO there is WorkspaceList.tempDir but no API to make other variants + private static String getFilePathSuffix() { + return System.getProperty(WorkspaceList.class.getName(), "@"); + } + + protected abstract static class SCMBasedRetrieverDescriptor extends LibraryRetrieverDescriptor { + + @POST + public FormValidation doCheckLibraryPath(@QueryParameter String libraryPath) { + libraryPath = Util.fixEmptyAndTrim(libraryPath); + if (libraryPath == null) { + return FormValidation.ok(); + } else if (PROHIBITED_DOUBLE_DOT.matcher(libraryPath).matches()) { + return FormValidation.error(Messages.SCMSourceRetriever_library_path_no_double_dot()); + } + return FormValidation.ok(); + } + + } + + @Restricted(DoNotUse.class) + @Extension + public static class WorkspaceListener extends ItemListener { + + @Override + public void onDeleted(Item item) { + deleteLibsDir(item, item.getFullName()); + } + + @Override + public void onLocationChanged(Item item, String oldFullName, String newFullName) { + deleteLibsDir(item, oldFullName); + } + + private static void deleteLibsDir(Item item, String itemFullName) { + if (item instanceof Job + && item.getClass() + .getName() + .equals("org.jenkinsci.plugins.workflow.job.WorkflowJob")) { + synchronized (item) { + String base = + expandVariablesForDirectory( + Jenkins.get().getRawWorkspaceDir(), + itemFullName, + item.getRootDir().getPath()); + FilePath dir = + new FilePath(new File(base)).withSuffix(getFilePathSuffix() + "libs"); + try { + if (dir.isDirectory()) { + LOGGER.log( + Level.INFO, + () -> "Deleting obsolete library workspace " + dir); + dir.deleteRecursive(); + } + } catch (IOException | InterruptedException e) { + LOGGER.log( + Level.WARNING, + e, + () -> "Could not delete obsolete library workspace " + dir); + } + } + } + } + + private static String expandVariablesForDirectory( + String base, String itemFullName, String itemRootDir) { + // If the item is moved, it is too late to look up its original workspace location by + // the time we get the notification. See: + // https://github.com/jenkinsci/jenkins/blob/f03183ab09ce5fb8f9f4cc9ccee42a3c3e6b2d3e/core/src/main/java/jenkins/model/Jenkins.java#L2567-L2576 + Map properties = new HashMap<>(); + properties.put("JENKINS_HOME", Jenkins.get().getRootDir().getPath()); + properties.put("ITEM_ROOTDIR", itemRootDir); + properties.put("ITEM_FULLNAME", itemFullName); // legacy, deprecated + properties.put( + "ITEM_FULL_NAME", itemFullName.replace(':', '$')); // safe, see JENKINS-12251 + return Util.replaceMacro(base, Collections.unmodifiableMap(properties)); + } + } + +} diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMRetriever.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMRetriever.java index da01ef44..2b01b819 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMRetriever.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMRetriever.java @@ -39,34 +39,20 @@ import hudson.util.FormValidation; import java.util.ArrayList; import java.util.List; -import edu.umd.cs.findbugs.annotations.CheckForNull; import jenkins.model.Jenkins; import org.jenkinsci.Symbol; import org.kohsuke.accmod.Restricted; import org.kohsuke.accmod.restrictions.DoNotUse; import org.kohsuke.accmod.restrictions.NoExternalUse; import org.kohsuke.stapler.DataBoundConstructor; -import org.kohsuke.stapler.DataBoundSetter; -import org.kohsuke.stapler.QueryParameter; -import org.kohsuke.stapler.verb.POST; /** * Uses legacy {@link SCM} to check out sources based on variable interpolation. */ -public class SCMRetriever extends LibraryRetriever { +public class SCMRetriever extends SCMBasedRetriever { private final SCM scm; - /** - * The path to the library inside of the SCM. - * - * {@code null} is the default and means that the library is in the root of the repository. Otherwise, the value is - * considered to be a relative path inside of the repository and always ends in a forward slash - * - * @see #setLibraryPath - */ - private @CheckForNull String libraryPath; - @DataBoundConstructor public SCMRetriever(SCM scm) { this.scm = scm; } @@ -75,25 +61,12 @@ public SCM getScm() { return scm; } - public String getLibraryPath() { - return libraryPath; - } - - @DataBoundSetter - public void setLibraryPath(String libraryPath) { - libraryPath = Util.fixEmptyAndTrim(libraryPath); - if (libraryPath != null && !libraryPath.endsWith("/")) { - libraryPath += '/'; - } - this.libraryPath = libraryPath; - } - @Override public void retrieve(String name, String version, boolean changelog, FilePath target, Run run, TaskListener listener) throws Exception { - SCMSourceRetriever.doRetrieve(name, changelog, scm, libraryPath, target, run, listener); + doRetrieve(name, changelog, scm, target, run, listener); } @Override public void retrieve(String name, String version, FilePath target, Run run, TaskListener listener) throws Exception { - SCMSourceRetriever.doRetrieve(name, true, scm, libraryPath, target, run, listener); + retrieve(name, version, true, target, run, listener); } @Override public FormValidation validateVersion(String name, String version, Item context) { @@ -105,12 +78,7 @@ public void setLibraryPath(String libraryPath) { } @Symbol("legacySCM") - @Extension(ordinal=-100) public static class DescriptorImpl extends LibraryRetrieverDescriptor { - - @POST - public FormValidation doCheckLibraryPath(@QueryParameter String libraryPath) { - return SCMSourceRetriever.DescriptorImpl.checkLibraryPath(libraryPath); - } + @Extension(ordinal=-100) public static class DescriptorImpl extends SCMBasedRetrieverDescriptor { @Override public String getDisplayName() { return "Legacy SCM"; diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever.java index 8b42c8f0..4ba9c5a6 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever.java @@ -24,93 +24,42 @@ package org.jenkinsci.plugins.workflow.libs; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import hudson.AbortException; import hudson.Extension; import hudson.ExtensionList; import hudson.FilePath; -import hudson.Functions; import hudson.Util; -import hudson.model.Computer; import hudson.model.Descriptor; import hudson.model.DescriptorVisibilityFilter; import hudson.model.Item; -import hudson.model.Job; -import hudson.model.Node; import hudson.model.Run; import hudson.model.TaskListener; -import hudson.model.TopLevelItem; -import hudson.model.listeners.ItemListener; -import hudson.scm.SCM; -import hudson.slaves.WorkspaceList; import hudson.util.FormValidation; import hudson.util.StreamTaskListener; -import java.io.File; -import java.io.IOException; -import java.io.InterruptedIOException; import java.io.StringWriter; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.Callable; -import java.util.logging.Level; -import java.util.logging.Logger; -import edu.umd.cs.findbugs.annotations.NonNull; -import edu.umd.cs.findbugs.annotations.CheckForNull; -import java.util.Set; -import java.util.TreeSet; -import jenkins.model.Jenkins; import jenkins.scm.api.SCMRevision; import jenkins.scm.api.SCMSource; import jenkins.scm.api.SCMSourceDescriptor; -import java.util.regex.Pattern; -import java.util.stream.Collectors; import org.jenkinsci.Symbol; import org.jenkinsci.plugins.structs.describable.CustomDescribableModel; import org.jenkinsci.plugins.structs.describable.UninstantiatedDescribable; -import org.jenkinsci.plugins.workflow.steps.scm.GenericSCMStep; -import org.jenkinsci.plugins.workflow.steps.scm.SCMStep; import org.kohsuke.accmod.Restricted; import org.kohsuke.accmod.restrictions.DoNotUse; import org.kohsuke.accmod.restrictions.NoExternalUse; import org.kohsuke.stapler.DataBoundConstructor; -import org.kohsuke.stapler.DataBoundSetter; -import org.kohsuke.stapler.QueryParameter; -import org.kohsuke.stapler.verb.POST; /** * Uses {@link SCMSource#fetch(String, TaskListener)} to retrieve a specific revision. */ -public class SCMSourceRetriever extends LibraryRetriever { - - private static final Logger LOGGER = Logger.getLogger(SCMSourceRetriever.class.getName()); - - @SuppressFBWarnings(value="MS_SHOULD_BE_FINAL", justification="Non-final for write access via the Script Console") - public static boolean INCLUDE_SRC_TEST_IN_LIBRARIES = Boolean.getBoolean(SCMSourceRetriever.class.getName() + ".INCLUDE_SRC_TEST_IN_LIBRARIES"); - /** - * Matches ".." in positions where it would be treated as the parent directory. - * - *

Used to prevent {@link #libraryPath} from being used for directory traversal. - */ - static final Pattern PROHIBITED_DOUBLE_DOT = Pattern.compile("(^|.*[\\\\/])\\.\\.($|[\\\\/].*)"); +public class SCMSourceRetriever extends SCMBasedRetriever { private final SCMSource scm; - private boolean clone; - - /** - * The path to the library inside of the SCM. - * - * {@code null} is the default and means that the library is in the root of the repository. Otherwise, the value is - * considered to be a relative path inside of the repository and always ends in a forward slash - * - * @see #setLibraryPath - */ - private @CheckForNull String libraryPath; - @DataBoundConstructor public SCMSourceRetriever(SCMSource scm) { this.scm = scm; } @@ -122,191 +71,18 @@ public SCMSource getScm() { return scm; } - public boolean isClone() { - return clone; - } - - @DataBoundSetter public void setClone(boolean clone) { - this.clone = clone; - } - - public String getLibraryPath() { - return libraryPath; - } - - @DataBoundSetter public void setLibraryPath(String libraryPath) { - libraryPath = Util.fixEmptyAndTrim(libraryPath); - if (libraryPath != null && !libraryPath.endsWith("/")) { - libraryPath += '/'; - } - this.libraryPath = libraryPath; - } - @Override public void retrieve(String name, String version, boolean changelog, FilePath target, Run run, TaskListener listener) throws Exception { SCMRevision revision = retrySCMOperation(listener, () -> scm.fetch(version, listener, run.getParent())); if (revision == null) { throw new AbortException("No version " + version + " found for library " + name); } - if (clone) { - if (changelog) { - listener.getLogger().println("WARNING: ignoring request to compute changelog in clone mode"); - } - doClone(scm.build(revision.getHead(), revision), libraryPath, target, run, listener); - } else { - doRetrieve(name, changelog, scm.build(revision.getHead(), revision), libraryPath, target, run, listener); - } + doRetrieve(name, changelog, scm.build(revision.getHead(), revision), target, run, listener); } @Override public void retrieve(String name, String version, FilePath target, Run run, TaskListener listener) throws Exception { retrieve(name, version, true, target, run, listener); } - private static T retrySCMOperation(TaskListener listener, Callable task) throws Exception{ - T ret = null; - for (int retryCount = Jenkins.get().getScmCheckoutRetryCount(); retryCount >= 0; retryCount--) { - try { - ret = task.call(); - break; - } - catch (AbortException e) { - // abort exception might have a null message. - // If so, just skip echoing it. - if (e.getMessage() != null) { - listener.error(e.getMessage()); - } - } - catch (InterruptedIOException e) { - throw e; - } - catch (Exception e) { - // checkout error not yet reported - Functions.printStackTrace(e, listener.error("Checkout failed")); - } - - if (retryCount == 0) // all attempts failed - throw new AbortException("Maximum checkout retry attempts reached, aborting"); - - listener.getLogger().println("Retrying after 10 seconds"); - Thread.sleep(10000); - } - return ret; - } - - static void doRetrieve(String name, boolean changelog, @NonNull SCM scm, String libraryPath, FilePath target, Run run, TaskListener listener) throws Exception { - // Adapted from CpsScmFlowDefinition: - SCMStep delegate = new GenericSCMStep(scm); - delegate.setPoll(false); // TODO we have no API for determining if a given SCMHead is branch-like or tag-like; would we want to turn on polling if the former? - delegate.setChangelog(changelog); - FilePath dir; - Node node = Jenkins.get(); - if (run.getParent() instanceof TopLevelItem) { - FilePath baseWorkspace = node.getWorkspaceFor((TopLevelItem) run.getParent()); - if (baseWorkspace == null) { - throw new IOException(node.getDisplayName() + " may be offline"); - } - String checkoutDirName = LibraryRecord.directoryNameFor(scm.getKey()); - dir = baseWorkspace.withSuffix(getFilePathSuffix() + "libs").child(checkoutDirName); - } else { // should not happen, but just in case: - throw new AbortException("Cannot check out in non-top-level build"); - } - Computer computer = node.toComputer(); - if (computer == null) { - throw new IOException(node.getDisplayName() + " may be offline"); - } - try (WorkspaceList.Lease lease = computer.getWorkspaceList().allocate(dir)) { - // Write the SCM key to a file as a debugging aid. - lease.path.withSuffix("-scm-key.txt").write(scm.getKey(), "UTF-8"); - retrySCMOperation(listener, () -> { - delegate.checkout(run, lease.path, listener, node.createLauncher(listener)); - return null; - }); - if (libraryPath == null) { - libraryPath = "."; - } else if (PROHIBITED_DOUBLE_DOT.matcher(libraryPath).matches()) { - throw new AbortException("Library path may not contain '..'"); - } - String excludes = INCLUDE_SRC_TEST_IN_LIBRARIES ? null : "src/test/"; - if (lease.path.child(libraryPath).child("src/test").exists()) { - listener.getLogger().println("Excluding src/test/ from checkout of " + scm.getKey() + " so that library test code cannot be accessed by Pipelines."); - listener.getLogger().println("To remove this log message, move the test code outside of src/. To restore the previous behavior that allowed access to files in src/test/, pass -D" + SCMSourceRetriever.class.getName() + ".INCLUDE_SRC_TEST_IN_LIBRARIES=true to the java command used to start Jenkins."); - } - // Cannot add WorkspaceActionImpl to private CpsFlowExecution.flowStartNodeActions; do we care? - // Copy sources with relevant files from the checkout: - lease.path.child(libraryPath).copyRecursiveTo("src/**/*.groovy,vars/*.groovy,vars/*.txt,resources/", excludes, target); - } - } - - // TODO there is WorkspaceList.tempDir but no API to make other variants - private static String getFilePathSuffix() { - return System.getProperty(WorkspaceList.class.getName(), "@"); - } - - /** - * Similar to {@link #doRetrieve} but used in {@link #clone} mode. - */ - private static void doClone(@NonNull SCM scm, String libraryPath, FilePath target, Run run, TaskListener listener) throws Exception { - SCMStep delegate = new GenericSCMStep(scm); - delegate.setPoll(false); - delegate.setChangelog(false); - if (libraryPath == null) { - retrySCMOperation(listener, () -> { - delegate.checkout(run, target, listener, Jenkins.get().createLauncher(listener)); - return null; - }); - } else { - if (PROHIBITED_DOUBLE_DOT.matcher(libraryPath).matches()) { - throw new AbortException("Library path may not contain '..'"); - } - FilePath root = target.child("root"); - retrySCMOperation(listener, () -> { - delegate.checkout(run, root, listener, Jenkins.get().createLauncher(listener)); - return null; - }); - FilePath subdir = root.child(libraryPath); - if (!subdir.isDirectory()) { - throw new AbortException("Did not find " + libraryPath + " in checkout"); - } - for (String content : List.of("src", "vars", "resources")) { - FilePath contentDir = subdir.child(content); - if (contentDir.isDirectory()) { - listener.getLogger().println("Moving " + content + " to top level"); - contentDir.renameTo(target.child(content)); - } - } - // root itself will be deleted below - } - Set deleted = new TreeSet<>(); - if (!INCLUDE_SRC_TEST_IN_LIBRARIES) { - FilePath srcTest = target.child("src/test"); - if (srcTest.isDirectory()) { - listener.getLogger().println("Excluding src/test/ from checkout of " + scm.getKey() + " so that library test code cannot be accessed by Pipelines."); - listener.getLogger().println("To remove this log message, move the test code outside of src/. To restore the previous behavior that allowed access to files in src/test/, pass -D" + SCMSourceRetriever.class.getName() + ".INCLUDE_SRC_TEST_IN_LIBRARIES=true to the java command used to start Jenkins."); - srcTest.deleteRecursive(); - deleted.add("src/test"); - } - } - for (FilePath child : target.list()) { - String name = child.getName(); - switch (name) { - case "src": - // TODO delete everything that is not *.groovy - break; - case "vars": - // TODO delete everything that is not *.groovy or *.txt, incl. subdirs - break; - case "resources": - // OK, leave it all - break; - default: - deleted.add(name); - child.deleteRecursive(); - } - } - if (!deleted.isEmpty()) { - listener.getLogger().println("Deleted " + deleted.stream().collect(Collectors.joining(", "))); - } - } - @Override public FormValidation validateVersion(String name, String version, Item context) { StringWriter w = new StringWriter(); try { @@ -325,22 +101,7 @@ private static void doClone(@NonNull SCM scm, String libraryPath, FilePath targe } @Symbol("modernSCM") - @Extension public static class DescriptorImpl extends LibraryRetrieverDescriptor implements CustomDescribableModel { - - static FormValidation checkLibraryPath(@QueryParameter String libraryPath) { - libraryPath = Util.fixEmptyAndTrim(libraryPath); - if (libraryPath == null) { - return FormValidation.ok(); - } else if (PROHIBITED_DOUBLE_DOT.matcher(libraryPath).matches()) { - return FormValidation.error(Messages.SCMSourceRetriever_library_path_no_double_dot()); - } - return FormValidation.ok(); - } - - @POST - public FormValidation doCheckLibraryPath(@QueryParameter String libraryPath) { - return checkLibraryPath(libraryPath); - } + @Extension public static class DescriptorImpl extends SCMBasedRetrieverDescriptor implements CustomDescribableModel { @Override public String getDisplayName() { return "Modern SCM"; @@ -389,62 +150,4 @@ public Collection getSCMDescriptors() { } - @Restricted(DoNotUse.class) - @Extension - public static class WorkspaceListener extends ItemListener { - - @Override - public void onDeleted(Item item) { - deleteLibsDir(item, item.getFullName()); - } - - @Override - public void onLocationChanged(Item item, String oldFullName, String newFullName) { - deleteLibsDir(item, oldFullName); - } - - private static void deleteLibsDir(Item item, String itemFullName) { - if (item instanceof Job - && item.getClass() - .getName() - .equals("org.jenkinsci.plugins.workflow.job.WorkflowJob")) { - synchronized (item) { - String base = - expandVariablesForDirectory( - Jenkins.get().getRawWorkspaceDir(), - itemFullName, - item.getRootDir().getPath()); - FilePath dir = - new FilePath(new File(base)).withSuffix(getFilePathSuffix() + "libs"); - try { - if (dir.isDirectory()) { - LOGGER.log( - Level.INFO, - () -> "Deleting obsolete library workspace " + dir); - dir.deleteRecursive(); - } - } catch (IOException | InterruptedException e) { - LOGGER.log( - Level.WARNING, - e, - () -> "Could not delete obsolete library workspace " + dir); - } - } - } - } - - private static String expandVariablesForDirectory( - String base, String itemFullName, String itemRootDir) { - // If the item is moved, it is too late to look up its original workspace location by - // the time we get the notification. See: - // https://github.com/jenkinsci/jenkins/blob/f03183ab09ce5fb8f9f4cc9ccee42a3c3e6b2d3e/core/src/main/java/jenkins/model/Jenkins.java#L2567-L2576 - Map properties = new HashMap<>(); - properties.put("JENKINS_HOME", Jenkins.get().getRootDir().getPath()); - properties.put("ITEM_ROOTDIR", itemRootDir); - properties.put("ITEM_FULLNAME", itemFullName); // legacy, deprecated - properties.put( - "ITEM_FULL_NAME", itemFullName.replace(':', '$')); // safe, see JENKINS-12251 - return Util.replaceMacro(base, Collections.unmodifiableMap(properties)); - } - } } diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/help-clone.html b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/help-clone.html similarity index 100% rename from src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/help-clone.html rename to src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/help-clone.html diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMRetriever/help-libraryPath.html b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/help-libraryPath.html similarity index 100% rename from src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMRetriever/help-libraryPath.html rename to src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/help-libraryPath.html diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/options.jelly b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/options.jelly new file mode 100644 index 00000000..97b6739e --- /dev/null +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/options.jelly @@ -0,0 +1,34 @@ + + + + + + + + + + + + diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/options.properties b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/options.properties new file mode 100644 index 00000000..5afdfe40 --- /dev/null +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/options.properties @@ -0,0 +1,23 @@ +# The MIT License +# +# Copyright 2020 CloudBees, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +libraryPath=Library Path (optional) diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMRetriever/config.jelly b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMRetriever/config.jelly index d7489eb1..d1b4cb02 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMRetriever/config.jelly +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMRetriever/config.jelly @@ -24,10 +24,8 @@ THE SOFTWARE. --> - + ${%blurb} - - - + diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMRetriever/config.properties b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMRetriever/config.properties index 9a9e0443..5d792f06 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMRetriever/config.properties +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMRetriever/config.properties @@ -20,7 +20,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -libraryPath=Library Path (optional) blurb=\ Loads a library from any SCM plugin supported by Jenkins Pipeline, for example in the checkout step. \ Not all version expressions may be supported, no prevalidation of versions is available, \ diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/config.jelly b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/config.jelly index 331ea146..d1b4cb02 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/config.jelly +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/config.jelly @@ -24,13 +24,8 @@ THE SOFTWARE. --> - + ${%blurb} - - - - - - + diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/config.properties b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/config.properties index 85d2d1d8..16109f36 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/config.properties +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/config.properties @@ -20,7 +20,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -libraryPath=Library Path (optional) blurb=\ Loads a library from an SCM plugin using newer interfaces optimized for this purpose. \ The recommended option when available. diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/help-libraryPath.html b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/help-libraryPath.html deleted file mode 100644 index acbba272..00000000 --- a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/help-libraryPath.html +++ /dev/null @@ -1,5 +0,0 @@ -

- A relative path from the root of the SCM to the root of the library. - Leave this field blank if the root of the library is the root of the SCM. - Note that ".." is not permitted as a path component to avoid security issues. -
diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetrieverTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetrieverTest.java index 91806f00..6ed2e412 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetrieverTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetrieverTest.java @@ -82,7 +82,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.matchesPattern; import static org.hamcrest.Matchers.nullValue; -import static org.jenkinsci.plugins.workflow.libs.SCMSourceRetriever.PROHIBITED_DOUBLE_DOT; +import static org.jenkinsci.plugins.workflow.libs.SCMBasedRetriever.PROHIBITED_DOUBLE_DOT; import static org.junit.Assume.assumeFalse; import org.jvnet.hudson.test.FlagRule; @@ -92,7 +92,7 @@ public class SCMSourceRetrieverTest { @Rule public JenkinsRule r = new JenkinsRule(); @Rule public GitSampleRepoRule sampleRepo = new GitSampleRepoRule(); @Rule public SubversionSampleRepoRule sampleRepoSvn = new SubversionSampleRepoRule(); - @Rule public FlagRule includeSrcTest = new FlagRule<>(() -> SCMSourceRetriever.INCLUDE_SRC_TEST_IN_LIBRARIES, v -> SCMSourceRetriever.INCLUDE_SRC_TEST_IN_LIBRARIES = v); + @Rule public FlagRule includeSrcTest = new FlagRule<>(() -> SCMBasedRetriever.INCLUDE_SRC_TEST_IN_LIBRARIES, v -> SCMBasedRetriever.INCLUDE_SRC_TEST_IN_LIBRARIES = v); @Issue("JENKINS-40408") @Test public void lease() throws Exception { @@ -444,7 +444,7 @@ public static class BasicSCMSource extends SCMSource { GlobalLibraries.get().setLibraries(Collections.singletonList(lc)); WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); p.setDefinition(new CpsFlowDefinition("@Library('echoing@master') import myecho; myecho()", true)); - SCMSourceRetriever.INCLUDE_SRC_TEST_IN_LIBRARIES = false; + SCMBasedRetriever.INCLUDE_SRC_TEST_IN_LIBRARIES = false; WorkflowRun b = r.buildAndAssertSuccess(p); assertFalse(r.jenkins.getWorkspaceFor(p).withSuffix("@libs").isDirectory()); r.assertLogContains("something special", b); @@ -465,7 +465,7 @@ public static class BasicSCMSource extends SCMSource { GlobalLibraries.get().setLibraries(Collections.singletonList(lc)); WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); p.setDefinition(new CpsFlowDefinition("@Library('echoing@master') import myecho; myecho()", true)); - SCMSourceRetriever.INCLUDE_SRC_TEST_IN_LIBRARIES = true; + SCMBasedRetriever.INCLUDE_SRC_TEST_IN_LIBRARIES = true; WorkflowRun b = r.buildAndAssertSuccess(p); assertFalse(r.jenkins.getWorkspaceFor(p).withSuffix("@libs").isDirectory()); r.assertLogContains("got something special", b); From 4a8971b1ad8b758a74561a057b78428ee695ae54 Mon Sep 17 00:00:00 2001 From: Jesse Glick Date: Fri, 24 Mar 2023 09:14:50 -0400 Subject: [PATCH 3/4] More advice on `clone`, backporting https://github.com/jenkinsci/pipeline-groovy-lib-plugin/pull/57/files/b7bd7741e05e751584afd78b1dc6f47dd06d4129..698ae6887eae199e4b5da9b478f5039265562350#diff-d9ea39311c121ef001c2131af309031434afafb5a19bd5961747f2c1f7082768R399 --- .../workflow/libs/SCMBasedRetriever/help-clone.html | 4 +++- .../plugins/workflow/libs/SCMSourceRetrieverTest.java | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/help-clone.html b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/help-clone.html index 2f375430..c0d502c7 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/help-clone.html +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/help-clone.html @@ -1,6 +1,8 @@
If checked, every build performs a fresh clone of the SCM rather than locking and updating a common copy. No changelog will be computed. - For Git, you are advised to select Advanced clone behaviors » Shallow clone to make the clone much faster. + For Git, you are advised to add Advanced clone behaviors + and then check Shallow clone and Honor refspec on initial clone and uncheck Fetch tags + to make the clone much faster. You may still enable Cache fetched versions on controller for quick retrieval if you prefer.
diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetrieverTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetrieverTest.java index 6ed2e412..a915e99a 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetrieverTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetrieverTest.java @@ -62,6 +62,7 @@ import static hudson.ExtensionList.lookupSingleton; import hudson.plugins.git.extensions.impl.CloneOption; import jenkins.plugins.git.traits.CloneOptionTrait; +import jenkins.plugins.git.traits.RefSpecsSCMSourceTrait; import jenkins.scm.api.trait.SCMSourceTrait; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -379,7 +380,9 @@ public static class BasicSCMSource extends SCMSource { sampleRepo.git("add", "."); sampleRepo.git("commit", "--message=init"); GitSCMSource src = new GitSCMSource(sampleRepo.toString()); - src.setTraits(List.of(new CloneOptionTrait(new CloneOption(true, null, null)))); + CloneOption cloneOption = new CloneOption(true, true, null, null); + cloneOption.setHonorRefspec(true); + src.setTraits(List.of(new CloneOptionTrait(cloneOption), new RefSpecsSCMSourceTrait("+refs/heads/master:refs/remotes/origin/master"))); SCMSourceRetriever scm = new SCMSourceRetriever(src); LibraryConfiguration lc = new LibraryConfiguration("echoing", scm); lc.setIncludeInChangesets(false); @@ -392,6 +395,8 @@ public static class BasicSCMSource extends SCMSource { r.assertLogContains("something special", b); r.assertLogContains("Deleted .git, README.md", b); r.assertLogContains("Using shallow clone with depth 1", b); + r.assertLogContains("Avoid fetching tags", b); + r.assertLogNotContains("+refs/heads/*:refs/remotes/origin/*", b); } @Test public void cloneModeLibraryPath() throws Exception { From 6d4bd490a28d4187b779d507810ddcd124013a27 Mon Sep 17 00:00:00 2001 From: Jesse Glick Date: Tue, 11 Apr 2023 15:36:40 -0400 Subject: [PATCH 4/4] Switching some clone-mode messages to fine logger https://github.com/jenkinsci/pipeline-groovy-lib-plugin/pull/55#discussion_r1163201588 --- .../plugins/workflow/libs/SCMBasedRetriever.java | 12 ++---------- .../workflow/libs/SCMSourceRetrieverTest.java | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever.java index cf6c75d1..d9f3a680 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever.java @@ -50,13 +50,10 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.TreeSet; import java.util.concurrent.Callable; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Pattern; -import java.util.stream.Collectors; import jenkins.model.Jenkins; import org.jenkinsci.plugins.workflow.steps.scm.GenericSCMStep; import org.jenkinsci.plugins.workflow.steps.scm.SCMStep; @@ -149,20 +146,18 @@ protected final void doRetrieve(String name, boolean changelog, @NonNull SCM scm for (String content : List.of("src", "vars", "resources")) { FilePath contentDir = subdir.child(content); if (contentDir.isDirectory()) { - listener.getLogger().println("Moving " + content + " to top level"); + LOGGER.fine(() -> "Moving " + content + " to top level in " + target); contentDir.renameTo(target.child(content)); } } // root itself will be deleted below } - Set deleted = new TreeSet<>(); if (!INCLUDE_SRC_TEST_IN_LIBRARIES) { FilePath srcTest = target.child("src/test"); if (srcTest.isDirectory()) { listener.getLogger().println("Excluding src/test/ from checkout of " + scm.getKey() + " so that library test code cannot be accessed by Pipelines."); listener.getLogger().println("To remove this log message, move the test code outside of src/. To restore the previous behavior that allowed access to files in src/test/, pass -D" + SCMSourceRetriever.class.getName() + ".INCLUDE_SRC_TEST_IN_LIBRARIES=true to the java command used to start Jenkins."); srcTest.deleteRecursive(); - deleted.add("src/test"); } } for (FilePath child : target.list()) { @@ -178,13 +173,10 @@ protected final void doRetrieve(String name, boolean changelog, @NonNull SCM scm // OK, leave it all break; default: - deleted.add(subdir); child.deleteRecursive(); + LOGGER.fine(() -> "Deleted " + child); } } - if (!deleted.isEmpty()) { - listener.getLogger().println("Deleted " + deleted.stream().collect(Collectors.joining(", "))); - } } else { // !clone FilePath dir; if (run.getParent() instanceof TopLevelItem) { diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetrieverTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetrieverTest.java index a915e99a..0b6a1583 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetrieverTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetrieverTest.java @@ -43,6 +43,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.logging.Level; import jenkins.plugins.git.GitSCMSource; import jenkins.plugins.git.GitSampleRepoRule; import jenkins.scm.api.SCMHead; @@ -80,12 +81,15 @@ import org.jvnet.hudson.test.WithoutJenkins; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.matchesPattern; import static org.hamcrest.Matchers.nullValue; import static org.jenkinsci.plugins.workflow.libs.SCMBasedRetriever.PROHIBITED_DOUBLE_DOT; import static org.junit.Assume.assumeFalse; import org.jvnet.hudson.test.FlagRule; +import org.jvnet.hudson.test.LoggerRule; public class SCMSourceRetrieverTest { @@ -94,6 +98,7 @@ public class SCMSourceRetrieverTest { @Rule public GitSampleRepoRule sampleRepo = new GitSampleRepoRule(); @Rule public SubversionSampleRepoRule sampleRepoSvn = new SubversionSampleRepoRule(); @Rule public FlagRule includeSrcTest = new FlagRule<>(() -> SCMBasedRetriever.INCLUDE_SRC_TEST_IN_LIBRARIES, v -> SCMBasedRetriever.INCLUDE_SRC_TEST_IN_LIBRARIES = v); + @Rule public LoggerRule logging = new LoggerRule().record(SCMBasedRetriever.class, Level.FINE); @Issue("JENKINS-40408") @Test public void lease() throws Exception { @@ -393,10 +398,13 @@ public static class BasicSCMSource extends SCMSource { WorkflowRun b = r.buildAndAssertSuccess(p); assertFalse(r.jenkins.getWorkspaceFor(p).withSuffix("@libs").isDirectory()); r.assertLogContains("something special", b); - r.assertLogContains("Deleted .git, README.md", b); r.assertLogContains("Using shallow clone with depth 1", b); r.assertLogContains("Avoid fetching tags", b); r.assertLogNotContains("+refs/heads/*:refs/remotes/origin/*", b); + File[] libDirs = new File(b.getRootDir(), "libs").listFiles(File::isDirectory); + assertThat(libDirs, arrayWithSize(1)); + String[] entries = libDirs[0].list(); + assertThat(entries, arrayContainingInAnyOrder("vars")); } @Test public void cloneModeLibraryPath() throws Exception { @@ -414,8 +422,10 @@ public static class BasicSCMSource extends SCMSource { p.setDefinition(new CpsFlowDefinition("@Library('root_sub_path@master') import myecho; myecho()", true)); WorkflowRun b = r.buildAndAssertSuccess(p); r.assertLogContains("something special", b); - r.assertLogContains("Moving vars to top level", b); - r.assertLogContains("Deleted root", b); + File[] libDirs = new File(b.getRootDir(), "libs").listFiles(File::isDirectory); + assertThat(libDirs, arrayWithSize(1)); + String[] entries = libDirs[0].list(); + assertThat(entries, arrayContainingInAnyOrder("vars")); } @Test public void cloneModeLibraryPathSecurity() throws Exception {