diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever.java new file mode 100644 index 00000000..d9f3a680 --- /dev/null +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever.java @@ -0,0 +1,328 @@ +/* + * The MIT License + * + * Copyright 2023 CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package org.jenkinsci.plugins.workflow.libs; + +import edu.umd.cs.findbugs.annotations.CheckForNull; +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import hudson.AbortException; +import hudson.Extension; +import hudson.FilePath; +import hudson.Functions; +import hudson.Util; +import hudson.model.Computer; +import hudson.model.Item; +import hudson.model.Job; +import hudson.model.Node; +import hudson.model.Run; +import hudson.model.TaskListener; +import hudson.model.TopLevelItem; +import hudson.model.listeners.ItemListener; +import hudson.scm.SCM; +import hudson.slaves.WorkspaceList; +import hudson.util.FormValidation; +import java.io.File; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Pattern; +import jenkins.model.Jenkins; +import org.jenkinsci.plugins.workflow.steps.scm.GenericSCMStep; +import org.jenkinsci.plugins.workflow.steps.scm.SCMStep; +import org.kohsuke.accmod.Restricted; +import org.kohsuke.accmod.restrictions.DoNotUse; +import org.kohsuke.stapler.DataBoundSetter; +import org.kohsuke.stapler.QueryParameter; +import org.kohsuke.stapler.verb.POST; + +/** + * Functionality common to {@link SCMSourceRetriever} and {@link SCMRetriever}. + */ +public abstract class SCMBasedRetriever extends LibraryRetriever { + + private static final Logger LOGGER = Logger.getLogger(SCMBasedRetriever.class.getName()); + + @SuppressFBWarnings(value = "MS_SHOULD_BE_FINAL", justification = "Non-final for write access via the Script Console") + public static boolean INCLUDE_SRC_TEST_IN_LIBRARIES = Boolean.getBoolean(SCMSourceRetriever.class.getName() + ".INCLUDE_SRC_TEST_IN_LIBRARIES"); + + /** + * Matches ".." in positions where it would be treated as the parent directory. + * + *

Used to prevent {@link #libraryPath} from being used for directory traversal. + */ + static final Pattern PROHIBITED_DOUBLE_DOT = Pattern.compile("(^|.*[\\\\/])\\.\\.($|[\\\\/].*)"); + + private boolean clone; + + /** + * The path to the library inside of the SCM. + * + * {@code null} is the default and means that the library is in the root of the repository. Otherwise, the value is + * considered to be a relative path inside of the repository and always ends in a forward slash + * + * @see #setLibraryPath + */ + private @CheckForNull String libraryPath; + + public boolean isClone() { + return clone; + } + + @DataBoundSetter public void setClone(boolean clone) { + this.clone = clone; + } + + public String getLibraryPath() { + return libraryPath; + } + + @DataBoundSetter public void setLibraryPath(String libraryPath) { + libraryPath = Util.fixEmptyAndTrim(libraryPath); + if (libraryPath != null && !libraryPath.endsWith("/")) { + libraryPath += '/'; + } + this.libraryPath = libraryPath; + } + + protected final void doRetrieve(String name, boolean changelog, @NonNull SCM scm, FilePath target, Run run, TaskListener listener) throws Exception { + if (libraryPath != null && PROHIBITED_DOUBLE_DOT.matcher(libraryPath).matches()) { + throw new AbortException("Library path may not contain '..'"); + } + if (clone && changelog) { + listener.getLogger().println("WARNING: ignoring request to compute changelog in clone mode"); + changelog = false; + } + // Adapted from CpsScmFlowDefinition: + SCMStep delegate = new GenericSCMStep(scm); + delegate.setPoll(false); // TODO we have no API for determining if a given SCMHead is branch-like or tag-like; would we want to turn on polling if the former? + delegate.setChangelog(changelog); + Node node = Jenkins.get(); + if (clone) { + if (libraryPath == null) { + retrySCMOperation(listener, () -> { + delegate.checkout(run, target, listener, Jenkins.get().createLauncher(listener)); + WorkspaceList.tempDir(target).deleteRecursive(); + return null; + }); + } else { + FilePath root = target.child("root"); + retrySCMOperation(listener, () -> { + delegate.checkout(run, root, listener, Jenkins.get().createLauncher(listener)); + WorkspaceList.tempDir(root).deleteRecursive(); + return null; + }); + FilePath subdir = root.child(libraryPath); + if (!subdir.isDirectory()) { + throw new AbortException("Did not find " + libraryPath + " in checkout"); + } + for (String content : List.of("src", "vars", "resources")) { + FilePath contentDir = subdir.child(content); + if (contentDir.isDirectory()) { + LOGGER.fine(() -> "Moving " + content + " to top level in " + target); + contentDir.renameTo(target.child(content)); + } + } + // root itself will be deleted below + } + if (!INCLUDE_SRC_TEST_IN_LIBRARIES) { + FilePath srcTest = target.child("src/test"); + if (srcTest.isDirectory()) { + listener.getLogger().println("Excluding src/test/ from checkout of " + scm.getKey() + " so that library test code cannot be accessed by Pipelines."); + listener.getLogger().println("To remove this log message, move the test code outside of src/. To restore the previous behavior that allowed access to files in src/test/, pass -D" + SCMSourceRetriever.class.getName() + ".INCLUDE_SRC_TEST_IN_LIBRARIES=true to the java command used to start Jenkins."); + srcTest.deleteRecursive(); + } + } + for (FilePath child : target.list()) { + String subdir = child.getName(); + switch (subdir) { + case "src": + // TODO delete everything that is not *.groovy + break; + case "vars": + // TODO delete everything that is not *.groovy or *.txt, incl. subdirs + break; + case "resources": + // OK, leave it all + break; + default: + child.deleteRecursive(); + LOGGER.fine(() -> "Deleted " + child); + } + } + } else { // !clone + FilePath dir; + if (run.getParent() instanceof TopLevelItem) { + FilePath baseWorkspace = node.getWorkspaceFor((TopLevelItem) run.getParent()); + if (baseWorkspace == null) { + throw new IOException(node.getDisplayName() + " may be offline"); + } + String checkoutDirName = LibraryRecord.directoryNameFor(scm.getKey()); + dir = baseWorkspace.withSuffix(getFilePathSuffix() + "libs").child(checkoutDirName); + } else { // should not happen, but just in case: + throw new AbortException("Cannot check out in non-top-level build"); + } + Computer computer = node.toComputer(); + if (computer == null) { + throw new IOException(node.getDisplayName() + " may be offline"); + } + try (WorkspaceList.Lease lease = computer.getWorkspaceList().allocate(dir)) { + // Write the SCM key to a file as a debugging aid. + lease.path.withSuffix("-scm-key.txt").write(scm.getKey(), "UTF-8"); + retrySCMOperation(listener, () -> { + delegate.checkout(run, lease.path, listener, node.createLauncher(listener)); + return null; + }); + if (libraryPath == null) { + libraryPath = "."; + } + String excludes = INCLUDE_SRC_TEST_IN_LIBRARIES ? null : "src/test/"; + if (lease.path.child(libraryPath).child("src/test").exists()) { + listener.getLogger().println("Excluding src/test/ from checkout of " + scm.getKey() + " so that library test code cannot be accessed by Pipelines."); + listener.getLogger().println("To remove this log message, move the test code outside of src/. To restore the previous behavior that allowed access to files in src/test/, pass -D" + SCMSourceRetriever.class.getName() + ".INCLUDE_SRC_TEST_IN_LIBRARIES=true to the java command used to start Jenkins."); + } + // Cannot add WorkspaceActionImpl to private CpsFlowExecution.flowStartNodeActions; do we care? + // Copy sources with relevant files from the checkout: + lease.path.child(libraryPath).copyRecursiveTo("src/**/*.groovy,vars/*.groovy,vars/*.txt,resources/", excludes, target); + } + } + } + + protected static T retrySCMOperation(TaskListener listener, Callable task) throws Exception{ + T ret = null; + for (int retryCount = Jenkins.get().getScmCheckoutRetryCount(); retryCount >= 0; retryCount--) { + try { + ret = task.call(); + break; + } + catch (AbortException e) { + // abort exception might have a null message. + // If so, just skip echoing it. + if (e.getMessage() != null) { + listener.error(e.getMessage()); + } + } + catch (InterruptedIOException e) { + throw e; + } + catch (Exception e) { + // checkout error not yet reported + Functions.printStackTrace(e, listener.error("Checkout failed")); + } + + if (retryCount == 0) // all attempts failed + throw new AbortException("Maximum checkout retry attempts reached, aborting"); + + listener.getLogger().println("Retrying after 10 seconds"); + Thread.sleep(10000); + } + return ret; + } + + // TODO there is WorkspaceList.tempDir but no API to make other variants + private static String getFilePathSuffix() { + return System.getProperty(WorkspaceList.class.getName(), "@"); + } + + protected abstract static class SCMBasedRetrieverDescriptor extends LibraryRetrieverDescriptor { + + @POST + public FormValidation doCheckLibraryPath(@QueryParameter String libraryPath) { + libraryPath = Util.fixEmptyAndTrim(libraryPath); + if (libraryPath == null) { + return FormValidation.ok(); + } else if (PROHIBITED_DOUBLE_DOT.matcher(libraryPath).matches()) { + return FormValidation.error(Messages.SCMSourceRetriever_library_path_no_double_dot()); + } + return FormValidation.ok(); + } + + } + + @Restricted(DoNotUse.class) + @Extension + public static class WorkspaceListener extends ItemListener { + + @Override + public void onDeleted(Item item) { + deleteLibsDir(item, item.getFullName()); + } + + @Override + public void onLocationChanged(Item item, String oldFullName, String newFullName) { + deleteLibsDir(item, oldFullName); + } + + private static void deleteLibsDir(Item item, String itemFullName) { + if (item instanceof Job + && item.getClass() + .getName() + .equals("org.jenkinsci.plugins.workflow.job.WorkflowJob")) { + synchronized (item) { + String base = + expandVariablesForDirectory( + Jenkins.get().getRawWorkspaceDir(), + itemFullName, + item.getRootDir().getPath()); + FilePath dir = + new FilePath(new File(base)).withSuffix(getFilePathSuffix() + "libs"); + try { + if (dir.isDirectory()) { + LOGGER.log( + Level.INFO, + () -> "Deleting obsolete library workspace " + dir); + dir.deleteRecursive(); + } + } catch (IOException | InterruptedException e) { + LOGGER.log( + Level.WARNING, + e, + () -> "Could not delete obsolete library workspace " + dir); + } + } + } + } + + private static String expandVariablesForDirectory( + String base, String itemFullName, String itemRootDir) { + // If the item is moved, it is too late to look up its original workspace location by + // the time we get the notification. See: + // https://github.com/jenkinsci/jenkins/blob/f03183ab09ce5fb8f9f4cc9ccee42a3c3e6b2d3e/core/src/main/java/jenkins/model/Jenkins.java#L2567-L2576 + Map properties = new HashMap<>(); + properties.put("JENKINS_HOME", Jenkins.get().getRootDir().getPath()); + properties.put("ITEM_ROOTDIR", itemRootDir); + properties.put("ITEM_FULLNAME", itemFullName); // legacy, deprecated + properties.put( + "ITEM_FULL_NAME", itemFullName.replace(':', '$')); // safe, see JENKINS-12251 + return Util.replaceMacro(base, Collections.unmodifiableMap(properties)); + } + } + +} diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMRetriever.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMRetriever.java index da01ef44..2b01b819 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMRetriever.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMRetriever.java @@ -39,34 +39,20 @@ import hudson.util.FormValidation; import java.util.ArrayList; import java.util.List; -import edu.umd.cs.findbugs.annotations.CheckForNull; import jenkins.model.Jenkins; import org.jenkinsci.Symbol; import org.kohsuke.accmod.Restricted; import org.kohsuke.accmod.restrictions.DoNotUse; import org.kohsuke.accmod.restrictions.NoExternalUse; import org.kohsuke.stapler.DataBoundConstructor; -import org.kohsuke.stapler.DataBoundSetter; -import org.kohsuke.stapler.QueryParameter; -import org.kohsuke.stapler.verb.POST; /** * Uses legacy {@link SCM} to check out sources based on variable interpolation. */ -public class SCMRetriever extends LibraryRetriever { +public class SCMRetriever extends SCMBasedRetriever { private final SCM scm; - /** - * The path to the library inside of the SCM. - * - * {@code null} is the default and means that the library is in the root of the repository. Otherwise, the value is - * considered to be a relative path inside of the repository and always ends in a forward slash - * - * @see #setLibraryPath - */ - private @CheckForNull String libraryPath; - @DataBoundConstructor public SCMRetriever(SCM scm) { this.scm = scm; } @@ -75,25 +61,12 @@ public SCM getScm() { return scm; } - public String getLibraryPath() { - return libraryPath; - } - - @DataBoundSetter - public void setLibraryPath(String libraryPath) { - libraryPath = Util.fixEmptyAndTrim(libraryPath); - if (libraryPath != null && !libraryPath.endsWith("/")) { - libraryPath += '/'; - } - this.libraryPath = libraryPath; - } - @Override public void retrieve(String name, String version, boolean changelog, FilePath target, Run run, TaskListener listener) throws Exception { - SCMSourceRetriever.doRetrieve(name, changelog, scm, libraryPath, target, run, listener); + doRetrieve(name, changelog, scm, target, run, listener); } @Override public void retrieve(String name, String version, FilePath target, Run run, TaskListener listener) throws Exception { - SCMSourceRetriever.doRetrieve(name, true, scm, libraryPath, target, run, listener); + retrieve(name, version, true, target, run, listener); } @Override public FormValidation validateVersion(String name, String version, Item context) { @@ -105,12 +78,7 @@ public void setLibraryPath(String libraryPath) { } @Symbol("legacySCM") - @Extension(ordinal=-100) public static class DescriptorImpl extends LibraryRetrieverDescriptor { - - @POST - public FormValidation doCheckLibraryPath(@QueryParameter String libraryPath) { - return SCMSourceRetriever.DescriptorImpl.checkLibraryPath(libraryPath); - } + @Extension(ordinal=-100) public static class DescriptorImpl extends SCMBasedRetrieverDescriptor { @Override public String getDisplayName() { return "Legacy SCM"; diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever.java index 9732508b..4ba9c5a6 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever.java @@ -24,88 +24,42 @@ package org.jenkinsci.plugins.workflow.libs; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import hudson.AbortException; import hudson.Extension; import hudson.ExtensionList; import hudson.FilePath; -import hudson.Functions; import hudson.Util; -import hudson.model.Computer; import hudson.model.Descriptor; import hudson.model.DescriptorVisibilityFilter; import hudson.model.Item; -import hudson.model.Job; -import hudson.model.Node; import hudson.model.Run; import hudson.model.TaskListener; -import hudson.model.TopLevelItem; -import hudson.model.listeners.ItemListener; -import hudson.scm.SCM; -import hudson.slaves.WorkspaceList; import hudson.util.FormValidation; import hudson.util.StreamTaskListener; -import java.io.File; -import java.io.IOException; -import java.io.InterruptedIOException; import java.io.StringWriter; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.Callable; -import java.util.logging.Level; -import java.util.logging.Logger; -import edu.umd.cs.findbugs.annotations.NonNull; -import edu.umd.cs.findbugs.annotations.CheckForNull; -import jenkins.model.Jenkins; import jenkins.scm.api.SCMRevision; import jenkins.scm.api.SCMSource; import jenkins.scm.api.SCMSourceDescriptor; -import java.util.regex.Pattern; import org.jenkinsci.Symbol; import org.jenkinsci.plugins.structs.describable.CustomDescribableModel; import org.jenkinsci.plugins.structs.describable.UninstantiatedDescribable; -import org.jenkinsci.plugins.workflow.steps.scm.GenericSCMStep; -import org.jenkinsci.plugins.workflow.steps.scm.SCMStep; import org.kohsuke.accmod.Restricted; import org.kohsuke.accmod.restrictions.DoNotUse; import org.kohsuke.accmod.restrictions.NoExternalUse; import org.kohsuke.stapler.DataBoundConstructor; -import org.kohsuke.stapler.DataBoundSetter; -import org.kohsuke.stapler.QueryParameter; -import org.kohsuke.stapler.verb.POST; /** * Uses {@link SCMSource#fetch(String, TaskListener)} to retrieve a specific revision. */ -public class SCMSourceRetriever extends LibraryRetriever { - - private static final Logger LOGGER = Logger.getLogger(SCMSourceRetriever.class.getName()); - - @SuppressFBWarnings(value="MS_SHOULD_BE_FINAL", justification="Non-final for write access via the Script Console") - public static boolean INCLUDE_SRC_TEST_IN_LIBRARIES = Boolean.getBoolean(SCMSourceRetriever.class.getName() + ".INCLUDE_SRC_TEST_IN_LIBRARIES"); - /** - * Matches ".." in positions where it would be treated as the parent directory. - * - *

Used to prevent {@link #libraryPath} from being used for directory traversal. - */ - static final Pattern PROHIBITED_DOUBLE_DOT = Pattern.compile("(^|.*[\\\\/])\\.\\.($|[\\\\/].*)"); +public class SCMSourceRetriever extends SCMBasedRetriever { private final SCMSource scm; - /** - * The path to the library inside of the SCM. - * - * {@code null} is the default and means that the library is in the root of the repository. Otherwise, the value is - * considered to be a relative path inside of the repository and always ends in a forward slash - * - * @see #setLibraryPath - */ - private @CheckForNull String libraryPath; - @DataBoundConstructor public SCMSourceRetriever(SCMSource scm) { this.scm = scm; } @@ -117,110 +71,18 @@ public SCMSource getScm() { return scm; } - public String getLibraryPath() { - return libraryPath; - } - - @DataBoundSetter public void setLibraryPath(String libraryPath) { - libraryPath = Util.fixEmptyAndTrim(libraryPath); - if (libraryPath != null && !libraryPath.endsWith("/")) { - libraryPath += '/'; - } - this.libraryPath = libraryPath; - } - @Override public void retrieve(String name, String version, boolean changelog, FilePath target, Run run, TaskListener listener) throws Exception { SCMRevision revision = retrySCMOperation(listener, () -> scm.fetch(version, listener, run.getParent())); if (revision == null) { throw new AbortException("No version " + version + " found for library " + name); } - doRetrieve(name, changelog, scm.build(revision.getHead(), revision), libraryPath, target, run, listener); + doRetrieve(name, changelog, scm.build(revision.getHead(), revision), target, run, listener); } @Override public void retrieve(String name, String version, FilePath target, Run run, TaskListener listener) throws Exception { retrieve(name, version, true, target, run, listener); } - private static T retrySCMOperation(TaskListener listener, Callable task) throws Exception{ - T ret = null; - for (int retryCount = Jenkins.get().getScmCheckoutRetryCount(); retryCount >= 0; retryCount--) { - try { - ret = task.call(); - break; - } - catch (AbortException e) { - // abort exception might have a null message. - // If so, just skip echoing it. - if (e.getMessage() != null) { - listener.error(e.getMessage()); - } - } - catch (InterruptedIOException e) { - throw e; - } - catch (Exception e) { - // checkout error not yet reported - Functions.printStackTrace(e, listener.error("Checkout failed")); - } - - if (retryCount == 0) // all attempts failed - throw new AbortException("Maximum checkout retry attempts reached, aborting"); - - listener.getLogger().println("Retrying after 10 seconds"); - Thread.sleep(10000); - } - return ret; - } - - static void doRetrieve(String name, boolean changelog, @NonNull SCM scm, String libraryPath, FilePath target, Run run, TaskListener listener) throws Exception { - // Adapted from CpsScmFlowDefinition: - SCMStep delegate = new GenericSCMStep(scm); - delegate.setPoll(false); // TODO we have no API for determining if a given SCMHead is branch-like or tag-like; would we want to turn on polling if the former? - delegate.setChangelog(changelog); - FilePath dir; - Node node = Jenkins.get(); - if (run.getParent() instanceof TopLevelItem) { - FilePath baseWorkspace = node.getWorkspaceFor((TopLevelItem) run.getParent()); - if (baseWorkspace == null) { - throw new IOException(node.getDisplayName() + " may be offline"); - } - String checkoutDirName = LibraryRecord.directoryNameFor(scm.getKey()); - dir = baseWorkspace.withSuffix(getFilePathSuffix() + "libs").child(checkoutDirName); - } else { // should not happen, but just in case: - throw new AbortException("Cannot check out in non-top-level build"); - } - Computer computer = node.toComputer(); - if (computer == null) { - throw new IOException(node.getDisplayName() + " may be offline"); - } - try (WorkspaceList.Lease lease = computer.getWorkspaceList().allocate(dir)) { - // Write the SCM key to a file as a debugging aid. - lease.path.withSuffix("-scm-key.txt").write(scm.getKey(), "UTF-8"); - retrySCMOperation(listener, () -> { - delegate.checkout(run, lease.path, listener, node.createLauncher(listener)); - return null; - }); - if (libraryPath == null) { - libraryPath = "."; - } else if (PROHIBITED_DOUBLE_DOT.matcher(libraryPath).matches()) { - throw new AbortException("Library path may not contain '..'"); - } - String excludes = INCLUDE_SRC_TEST_IN_LIBRARIES ? null : "src/test/"; - if (lease.path.child(libraryPath).child("src/test").exists()) { - listener.getLogger().println("Excluding src/test/ from checkout of " + scm.getKey() + " so that library test code cannot be accessed by Pipelines."); - listener.getLogger().println("To remove this log message, move the test code outside of src/. To restore the previous behavior that allowed access to files in src/test/, pass -D" + SCMSourceRetriever.class.getName() + ".INCLUDE_SRC_TEST_IN_LIBRARIES=true to the java command used to start Jenkins."); - } - // Cannot add WorkspaceActionImpl to private CpsFlowExecution.flowStartNodeActions; do we care? - // Copy sources with relevant files from the checkout: - lease.path.child(libraryPath).copyRecursiveTo("src/**/*.groovy,vars/*.groovy,vars/*.txt,resources/", excludes, target); - } - } - - // TODO there is WorkspaceList.tempDir but no API to make other variants - private static String getFilePathSuffix() { - return System.getProperty(WorkspaceList.class.getName(), "@"); - } - @Override public FormValidation validateVersion(String name, String version, Item context) { StringWriter w = new StringWriter(); try { @@ -239,22 +101,7 @@ private static String getFilePathSuffix() { } @Symbol("modernSCM") - @Extension public static class DescriptorImpl extends LibraryRetrieverDescriptor implements CustomDescribableModel { - - static FormValidation checkLibraryPath(@QueryParameter String libraryPath) { - libraryPath = Util.fixEmptyAndTrim(libraryPath); - if (libraryPath == null) { - return FormValidation.ok(); - } else if (PROHIBITED_DOUBLE_DOT.matcher(libraryPath).matches()) { - return FormValidation.error(Messages.SCMSourceRetriever_library_path_no_double_dot()); - } - return FormValidation.ok(); - } - - @POST - public FormValidation doCheckLibraryPath(@QueryParameter String libraryPath) { - return checkLibraryPath(libraryPath); - } + @Extension public static class DescriptorImpl extends SCMBasedRetrieverDescriptor implements CustomDescribableModel { @Override public String getDisplayName() { return "Modern SCM"; @@ -303,62 +150,4 @@ public Collection getSCMDescriptors() { } - @Restricted(DoNotUse.class) - @Extension - public static class WorkspaceListener extends ItemListener { - - @Override - public void onDeleted(Item item) { - deleteLibsDir(item, item.getFullName()); - } - - @Override - public void onLocationChanged(Item item, String oldFullName, String newFullName) { - deleteLibsDir(item, oldFullName); - } - - private static void deleteLibsDir(Item item, String itemFullName) { - if (item instanceof Job - && item.getClass() - .getName() - .equals("org.jenkinsci.plugins.workflow.job.WorkflowJob")) { - synchronized (item) { - String base = - expandVariablesForDirectory( - Jenkins.get().getRawWorkspaceDir(), - itemFullName, - item.getRootDir().getPath()); - FilePath dir = - new FilePath(new File(base)).withSuffix(getFilePathSuffix() + "libs"); - try { - if (dir.isDirectory()) { - LOGGER.log( - Level.INFO, - () -> "Deleting obsolete library workspace " + dir); - dir.deleteRecursive(); - } - } catch (IOException | InterruptedException e) { - LOGGER.log( - Level.WARNING, - e, - () -> "Could not delete obsolete library workspace " + dir); - } - } - } - } - - private static String expandVariablesForDirectory( - String base, String itemFullName, String itemRootDir) { - // If the item is moved, it is too late to look up its original workspace location by - // the time we get the notification. See: - // https://github.com/jenkinsci/jenkins/blob/f03183ab09ce5fb8f9f4cc9ccee42a3c3e6b2d3e/core/src/main/java/jenkins/model/Jenkins.java#L2567-L2576 - Map properties = new HashMap<>(); - properties.put("JENKINS_HOME", Jenkins.get().getRootDir().getPath()); - properties.put("ITEM_ROOTDIR", itemRootDir); - properties.put("ITEM_FULLNAME", itemFullName); // legacy, deprecated - properties.put( - "ITEM_FULL_NAME", itemFullName.replace(':', '$')); // safe, see JENKINS-12251 - return Util.replaceMacro(base, Collections.unmodifiableMap(properties)); - } - } } diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/help-clone.html b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/help-clone.html new file mode 100644 index 00000000..c0d502c7 --- /dev/null +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/help-clone.html @@ -0,0 +1,8 @@ +

+ If checked, every build performs a fresh clone of the SCM rather than locking and updating a common copy. + No changelog will be computed. + For Git, you are advised to add Advanced clone behaviors + and then check Shallow clone and Honor refspec on initial clone and uncheck Fetch tags + to make the clone much faster. + You may still enable Cache fetched versions on controller for quick retrieval if you prefer. +
diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMRetriever/help-libraryPath.html b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/help-libraryPath.html similarity index 100% rename from src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMRetriever/help-libraryPath.html rename to src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/help-libraryPath.html diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/options.jelly b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/options.jelly new file mode 100644 index 00000000..97b6739e --- /dev/null +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/options.jelly @@ -0,0 +1,34 @@ + + + + + + + + + + + + diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/options.properties b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/options.properties new file mode 100644 index 00000000..5afdfe40 --- /dev/null +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMBasedRetriever/options.properties @@ -0,0 +1,23 @@ +# The MIT License +# +# Copyright 2020 CloudBees, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +libraryPath=Library Path (optional) diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMRetriever/config.jelly b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMRetriever/config.jelly index d7489eb1..d1b4cb02 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMRetriever/config.jelly +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMRetriever/config.jelly @@ -24,10 +24,8 @@ THE SOFTWARE. --> - + ${%blurb} - - - + diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMRetriever/config.properties b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMRetriever/config.properties index 9a9e0443..5d792f06 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMRetriever/config.properties +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMRetriever/config.properties @@ -20,7 +20,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -libraryPath=Library Path (optional) blurb=\ Loads a library from any SCM plugin supported by Jenkins Pipeline, for example in the checkout step. \ Not all version expressions may be supported, no prevalidation of versions is available, \ diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/config.jelly b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/config.jelly index d7489eb1..d1b4cb02 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/config.jelly +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/config.jelly @@ -24,10 +24,8 @@ THE SOFTWARE. --> - + ${%blurb} - - - + diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/config.properties b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/config.properties index 85d2d1d8..16109f36 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/config.properties +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/config.properties @@ -20,7 +20,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -libraryPath=Library Path (optional) blurb=\ Loads a library from an SCM plugin using newer interfaces optimized for this purpose. \ The recommended option when available. diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/help-libraryPath.html b/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/help-libraryPath.html deleted file mode 100644 index acbba272..00000000 --- a/src/main/resources/org/jenkinsci/plugins/workflow/libs/SCMSourceRetriever/help-libraryPath.html +++ /dev/null @@ -1,5 +0,0 @@ -
- A relative path from the root of the SCM to the root of the library. - Leave this field blank if the root of the library is the root of the SCM. - Note that ".." is not permitted as a path component to avoid security issues. -
diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetrieverTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetrieverTest.java index e1b1553a..0b6a1583 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetrieverTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/SCMSourceRetrieverTest.java @@ -43,6 +43,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.logging.Level; import jenkins.plugins.git.GitSCMSource; import jenkins.plugins.git.GitSampleRepoRule; import jenkins.scm.api.SCMHead; @@ -60,6 +61,10 @@ import org.jenkinsci.plugins.workflow.job.WorkflowRun; import static hudson.ExtensionList.lookupSingleton; +import hudson.plugins.git.extensions.impl.CloneOption; +import jenkins.plugins.git.traits.CloneOptionTrait; +import jenkins.plugins.git.traits.RefSpecsSCMSourceTrait; +import jenkins.scm.api.trait.SCMSourceTrait; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.instanceOf; @@ -76,11 +81,15 @@ import org.jvnet.hudson.test.WithoutJenkins; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.matchesPattern; import static org.hamcrest.Matchers.nullValue; -import static org.jenkinsci.plugins.workflow.libs.SCMSourceRetriever.PROHIBITED_DOUBLE_DOT; +import static org.jenkinsci.plugins.workflow.libs.SCMBasedRetriever.PROHIBITED_DOUBLE_DOT; import static org.junit.Assume.assumeFalse; +import org.jvnet.hudson.test.FlagRule; +import org.jvnet.hudson.test.LoggerRule; public class SCMSourceRetrieverTest { @@ -88,6 +97,8 @@ public class SCMSourceRetrieverTest { @Rule public JenkinsRule r = new JenkinsRule(); @Rule public GitSampleRepoRule sampleRepo = new GitSampleRepoRule(); @Rule public SubversionSampleRepoRule sampleRepoSvn = new SubversionSampleRepoRule(); + @Rule public FlagRule includeSrcTest = new FlagRule<>(() -> SCMBasedRetriever.INCLUDE_SRC_TEST_IN_LIBRARIES, v -> SCMBasedRetriever.INCLUDE_SRC_TEST_IN_LIBRARIES = v); + @Rule public LoggerRule logging = new LoggerRule().record(SCMBasedRetriever.class, Level.FINE); @Issue("JENKINS-40408") @Test public void lease() throws Exception { @@ -366,6 +377,116 @@ public static class BasicSCMSource extends SCMSource { assertFalse(ws.exists()); } + @Test public void cloneMode() throws Exception { + sampleRepo.init(); + sampleRepo.write("vars/myecho.groovy", "def call() {echo 'something special'}"); + sampleRepo.write("README.md", "Summary"); + sampleRepo.git("rm", "file"); + sampleRepo.git("add", "."); + sampleRepo.git("commit", "--message=init"); + GitSCMSource src = new GitSCMSource(sampleRepo.toString()); + CloneOption cloneOption = new CloneOption(true, true, null, null); + cloneOption.setHonorRefspec(true); + src.setTraits(List.of(new CloneOptionTrait(cloneOption), new RefSpecsSCMSourceTrait("+refs/heads/master:refs/remotes/origin/master"))); + SCMSourceRetriever scm = new SCMSourceRetriever(src); + LibraryConfiguration lc = new LibraryConfiguration("echoing", scm); + lc.setIncludeInChangesets(false); + scm.setClone(true); + GlobalLibraries.get().setLibraries(Collections.singletonList(lc)); + WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); + p.setDefinition(new CpsFlowDefinition("@Library('echoing@master') import myecho; myecho()", true)); + WorkflowRun b = r.buildAndAssertSuccess(p); + assertFalse(r.jenkins.getWorkspaceFor(p).withSuffix("@libs").isDirectory()); + r.assertLogContains("something special", b); + r.assertLogContains("Using shallow clone with depth 1", b); + r.assertLogContains("Avoid fetching tags", b); + r.assertLogNotContains("+refs/heads/*:refs/remotes/origin/*", b); + File[] libDirs = new File(b.getRootDir(), "libs").listFiles(File::isDirectory); + assertThat(libDirs, arrayWithSize(1)); + String[] entries = libDirs[0].list(); + assertThat(entries, arrayContainingInAnyOrder("vars")); + } + + @Test public void cloneModeLibraryPath() throws Exception { + sampleRepo.init(); + sampleRepo.write("sub/path/vars/myecho.groovy", "def call() {echo 'something special'}"); + sampleRepo.git("add", "sub"); + sampleRepo.git("commit", "--message=init"); + SCMSourceRetriever scm = new SCMSourceRetriever(new GitSCMSource(sampleRepo.toString())); + LibraryConfiguration lc = new LibraryConfiguration("root_sub_path", scm); + lc.setIncludeInChangesets(false); + scm.setLibraryPath("sub/path/"); + scm.setClone(true); + GlobalLibraries.get().setLibraries(Collections.singletonList(lc)); + WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); + p.setDefinition(new CpsFlowDefinition("@Library('root_sub_path@master') import myecho; myecho()", true)); + WorkflowRun b = r.buildAndAssertSuccess(p); + r.assertLogContains("something special", b); + File[] libDirs = new File(b.getRootDir(), "libs").listFiles(File::isDirectory); + assertThat(libDirs, arrayWithSize(1)); + String[] entries = libDirs[0].list(); + assertThat(entries, arrayContainingInAnyOrder("vars")); + } + + @Test public void cloneModeLibraryPathSecurity() throws Exception { + sampleRepo.init(); + sampleRepo.write("sub/path/vars/myecho.groovy", "def call() {echo 'something special'}"); + sampleRepo.git("add", "sub"); + sampleRepo.git("commit", "--message=init"); + SCMSourceRetriever scm = new SCMSourceRetriever(new GitSCMSource(sampleRepo.toString())); + LibraryConfiguration lc = new LibraryConfiguration("root_sub_path", scm); + lc.setIncludeInChangesets(false); + scm.setLibraryPath("sub/path/../../../jenkins_home/foo"); + scm.setClone(true); + GlobalLibraries.get().setLibraries(Collections.singletonList(lc)); + WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); + p.setDefinition(new CpsFlowDefinition("@Library('root_sub_path@master') import myecho; myecho()", true)); + WorkflowRun b = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); + r.assertLogContains("Library path may not contain '..'", b); + } + + @Test public void cloneModeExcludeSrcTest() throws Exception { + sampleRepo.init(); + sampleRepo.write("vars/myecho.groovy", "def call() {echo 'something special'}"); + sampleRepo.write("src/test/X.groovy", "// irrelevant"); + sampleRepo.write("README.md", "Summary"); + sampleRepo.git("add", "."); + sampleRepo.git("commit", "--message=init"); + SCMSourceRetriever scm = new SCMSourceRetriever(new GitSCMSource(sampleRepo.toString())); + LibraryConfiguration lc = new LibraryConfiguration("echoing", scm); + lc.setIncludeInChangesets(false); + scm.setClone(true); + GlobalLibraries.get().setLibraries(Collections.singletonList(lc)); + WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); + p.setDefinition(new CpsFlowDefinition("@Library('echoing@master') import myecho; myecho()", true)); + SCMBasedRetriever.INCLUDE_SRC_TEST_IN_LIBRARIES = false; + WorkflowRun b = r.buildAndAssertSuccess(p); + assertFalse(r.jenkins.getWorkspaceFor(p).withSuffix("@libs").isDirectory()); + r.assertLogContains("something special", b); + r.assertLogContains("Excluding src/test/ from checkout", b); + } + + @Test public void cloneModeIncludeSrcTest() throws Exception { + sampleRepo.init(); + sampleRepo.write("vars/myecho.groovy", "def call() {echo(/got ${new test.X().m()}/)}"); + sampleRepo.write("src/test/X.groovy", "package test; class X {def m() {'something special'}}"); + sampleRepo.write("README.md", "Summary"); + sampleRepo.git("add", "."); + sampleRepo.git("commit", "--message=init"); + SCMSourceRetriever scm = new SCMSourceRetriever(new GitSCMSource(sampleRepo.toString())); + LibraryConfiguration lc = new LibraryConfiguration("echoing", scm); + lc.setIncludeInChangesets(false); + scm.setClone(true); + GlobalLibraries.get().setLibraries(Collections.singletonList(lc)); + WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); + p.setDefinition(new CpsFlowDefinition("@Library('echoing@master') import myecho; myecho()", true)); + SCMBasedRetriever.INCLUDE_SRC_TEST_IN_LIBRARIES = true; + WorkflowRun b = r.buildAndAssertSuccess(p); + assertFalse(r.jenkins.getWorkspaceFor(p).withSuffix("@libs").isDirectory()); + r.assertLogContains("got something special", b); + r.assertLogNotContains("Excluding src/test/ from checkout", b); + } + @Issue("SECURITY-2441") @Test public void libraryNamesAreNotUsedAsCheckoutDirectories() throws Exception { sampleRepo.init();