Merge branch 'master' into fix-cache-deletion

Author: dwnusbaum

.github/workflows/cd.yaml

@@ -43,11 +43,11 @@ jobs:
     if: needs.validate.outputs.should_release == 'true'
     steps:
     - name: Check out
-      uses: actions/checkout@v2.4.0
+      uses: actions/checkout@v3
       with:
         fetch-depth: 0
     - name: Set up JDK 8
-      uses: actions/setup-java@v2
+      uses: actions/setup-java@v3
       with:
         distribution: 'adopt'
         java-version: 8

pom.xml

@@ -28,7 +28,7 @@
     <parent>
         <groupId>org.jenkins-ci.plugins</groupId>
         <artifactId>plugin</artifactId>
-        <version>4.33</version>
+        <version>4.40</version>
         <relativePath />
     </parent>
     <groupId>org.jenkins-ci.plugins.workflow</groupId>
@@ -124,31 +124,36 @@
             <artifactId>workflow-support</artifactId>
         </dependency>
 
-        <!-- test only plugins -->
         <dependency>
-            <groupId>org.jenkins-ci.plugins.workflow</groupId>
-            <artifactId>workflow-support</artifactId>
-            <classifier>tests</classifier>
-            <scope>test</scope>
+            <groupId>org.jenkins-ci.plugins</groupId>
+            <artifactId>variant</artifactId>
         </dependency>
         <dependency>
             <groupId>org.jenkins-ci.plugins.workflow</groupId>
-            <artifactId>workflow-job</artifactId>
-            <scope>test</scope>
+            <artifactId>workflow-multibranch</artifactId>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.jenkins-ci.plugins</groupId>
+            <artifactId>branch-api</artifactId>
+            <optional>true</optional>
         </dependency>
+
+        <!-- test only plugins -->
         <dependency>
             <groupId>org.jenkins-ci.plugins.workflow</groupId>
-            <artifactId>workflow-basic-steps</artifactId>
+            <artifactId>workflow-support</artifactId>
+            <classifier>tests</classifier>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.jenkins-ci.plugins.workflow</groupId>
-            <artifactId>workflow-durable-task-step</artifactId>
+            <artifactId>workflow-basic-steps</artifactId>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.jenkins-ci.plugins.workflow</groupId>
-            <artifactId>workflow-multibranch</artifactId>
+            <artifactId>workflow-durable-task-step</artifactId>
             <scope>test</scope>
         </dependency>
         <dependency>
@@ -203,10 +208,5 @@
             <version>1.10.1</version>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>org.jenkins-ci.plugins</groupId>
-            <artifactId>branch-api</artifactId>
-            <scope>test</scope>
-        </dependency>
     </dependencies>
 </project>

src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java

@@ -17,6 +17,7 @@
 import java.util.Collections;
 import java.util.List;
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
 
 public final class LibraryCachingConfiguration extends AbstractDescribableImpl<LibraryCachingConfiguration> {
     private int refreshTimeMinutes;
@@ -56,7 +57,19 @@ private List<String> getExcludedVersions() {
     }
 
     public Boolean isExcluded(String version) {
-        return getExcludedVersions().contains(version);
+        // exit early if the version passed in is null or empty
+        if (StringUtils.isBlank(version)) {
+            return false;
+        }
+        for (String it : getExcludedVersions()) {
+            // confirm that the excluded versions aren't null or empty
+            // and if the version contains the exclusion thus it can be
+            // anywhere in the string.
+            if (StringUtils.isNotBlank(it) && version.contains(it)){
+                return true;
+            }
+        }
+        return false;
     }
 
     @Override public String toString() {
@@ -96,4 +109,4 @@ public FormValidation doClearCache(@QueryParameter String name) throws Interrupt
         }
 
     }
-}
+}

src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryStep.java

@@ -37,6 +37,7 @@
 import hudson.model.ItemGroup;
 import hudson.model.Run;
 import hudson.model.TaskListener;
+import hudson.scm.SCM;
 import hudson.security.AccessControlled;
 import java.io.File;
 import java.io.IOException;
@@ -60,6 +61,7 @@
 import edu.umd.cs.findbugs.annotations.NonNull;
 import javax.inject.Inject;
 import jenkins.model.Jenkins;
+import jenkins.scm.impl.SingleSCMSource;
 import org.codehaus.groovy.control.MultipleCompilationErrorsException;
 import org.codehaus.groovy.runtime.InvokerHelper;
 import org.jenkinsci.plugins.scriptsecurity.sandbox.whitelists.AbstractWhitelist;
@@ -192,6 +194,18 @@ public static class Execution extends AbstractSynchronousNonBlockingStepExecutio
             } else if (version == null) {
                 throw new AbortException("Must specify a version for library " + name);
             }
+            // When a user specifies a non-null retriever, they may be using SCMVar in its configuration,
+            // so we need to run MultibranchScmRevisionVerifier to prevent unsafe behavior.
+            // SCMVar would typically be used with SCMRetriever, but it is also possible to use it with SCMSourceRetriever and SingleSCMSource.
+            // There may be false-positive rejections if a Multibranch Pipeline for the repo of a Pipeline library
+            // uses the library step with a non-null retriever to check out a static version of the library.
+            // Fixing this would require us being able to detect usage of SCMVar precisely, which is not currently possible.
+            else if (retriever instanceof SCMRetriever) {
+                verifyRevision(((SCMRetriever) retriever).getScm(), name);
+            } else if (retriever instanceof SCMSourceRetriever && ((SCMSourceRetriever) retriever).getScm() instanceof SingleSCMSource) {
+                verifyRevision(((SingleSCMSource) ((SCMSourceRetriever) retriever).getScm()).getScm(), name);
+            }
+
             LibraryRecord record = new LibraryRecord(name, version, trusted, changelog, cachingConfiguration, source);
             LibrariesAction action = run.getAction(LibrariesAction.class);
             if (action == null) {
@@ -219,6 +233,12 @@ public static class Execution extends AbstractSynchronousNonBlockingStepExecutio
             return new LoadedClasses(name, record.getDirectoryName(), trusted, changelog, run);
         }
 
+        private void verifyRevision(SCM scm, String name) throws IOException, InterruptedException {
+            for (LibraryStepRetrieverVerifier revisionVerifier : LibraryStepRetrieverVerifier.all()) {
+                revisionVerifier.verify(this.run, listener, scm, name);
+            }
+        }
+
     }
 
     public static final class LoadedClasses extends GroovyObjectSupport implements Serializable {

src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryStepRetrieverVerifier.java

@@ -0,0 +1,21 @@
+package org.jenkinsci.plugins.workflow.libs;
+
+import hudson.ExtensionList;
+import hudson.ExtensionPoint;
+import hudson.model.Run;
+import hudson.model.TaskListener;
+import hudson.scm.SCM;
+import org.kohsuke.accmod.Restricted;
+import org.kohsuke.accmod.restrictions.NoExternalUse;
+
+import java.io.IOException;
+
+@Restricted(NoExternalUse.class)
+public interface LibraryStepRetrieverVerifier extends ExtensionPoint {
+
+    void verify(Run<?, ?> run, TaskListener listener, SCM scm, String name) throws IOException, InterruptedException;
+
+    static ExtensionList<LibraryStepRetrieverVerifier> all() {
+        return ExtensionList.lookup(LibraryStepRetrieverVerifier.class);
+    }
+}

src/main/java/org/jenkinsci/plugins/workflow/libs/MultibranchScmRevisionVerifier.java

@@ -0,0 +1,67 @@
+package org.jenkinsci.plugins.workflow.libs;
+
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import hudson.AbortException;
+import hudson.model.Job;
+import hudson.model.Run;
+import hudson.model.TaskListener;
+import hudson.scm.SCM;
+import jenkins.branch.Branch;
+import jenkins.scm.api.SCMHead;
+import jenkins.scm.api.SCMRevision;
+import jenkins.scm.api.SCMRevisionAction;
+import jenkins.scm.api.SCMSource;
+import jenkins.scm.api.SCMSourceOwner;
+import org.jenkinsci.plugins.variant.OptionalExtension;
+import org.jenkinsci.plugins.workflow.multibranch.BranchJobProperty;
+
+import java.io.IOException;
+
+@OptionalExtension(requirePlugins={"workflow-multibranch"})
+public class MultibranchScmRevisionVerifier implements LibraryStepRetrieverVerifier {
+
+    @SuppressFBWarnings("MS_SHOULD_BE_FINAL") // For script console and tests.
+    public static boolean DISABLED = Boolean.getBoolean(MultibranchScmRevisionVerifier.class.getName() + ".DISABLED");
+
+    /**
+     * Abort library retrieval if the specified build is from a Multibranch Pipeline configured to build the library's SCM and the revision being built is untrusted.
+     * Comparable to the defenses against untrusted users in {@code SCMBinder}, but here we care about the library rather than the Jenkinsfile.
+     * @throws AbortException if the specified build is from a Multibranch Pipeline configured to build the library's SCM and the revision being built is untrusted
+     */
+    @Override
+    public void verify(Run<?, ?> run, TaskListener listener, SCM libraryScm, String name) throws IOException, InterruptedException {
+        if (DISABLED) {
+            return;
+        }
+        // Adapted from ReadTrustedStep
+        Job<?, ?> job = run.getParent();
+        BranchJobProperty property = job.getProperty(BranchJobProperty.class);
+        if (property == null || !(job.getParent() instanceof SCMSourceOwner)) {
+            // Not a multibranch project, so we do not care.
+            // It is possible to use legacySCM(scm) from a non-multibranch Pipeline that uses CpsScmFlowDefinition,
+            // but in that case we implicitly trust the changes because only a user with Item/Configure permission can select which branches to build.
+            return;
+        }
+        Branch pipelineBranch = property.getBranch();
+        SCMSource pipelineScmSource = ((SCMSourceOwner)job.getParent()).getSCMSource(pipelineBranch.getSourceId());
+        if (pipelineScmSource == null) {
+            throw new IllegalStateException(pipelineBranch.getSourceId() + " not found");
+        }
+        SCMHead head = pipelineBranch.getHead();
+        SCMRevision headRevision;
+        SCMRevisionAction action = run.getAction(SCMRevisionAction.class);
+        if (action != null) {
+            headRevision = action.getRevision();
+        } else {
+            headRevision = pipelineScmSource.fetch(head, listener);
+            if (headRevision == null) {
+                throw new AbortException("Could not determine exact tip revision of " + pipelineBranch.getName());
+            }
+            run.addAction(new SCMRevisionAction(pipelineScmSource, headRevision));
+        }
+        SCMRevision trustedRevision = pipelineScmSource.getTrustedRevision(headRevision, listener);
+        if (!headRevision.equals(trustedRevision) && libraryScm.getKey().equals(pipelineScmSource.build(head, headRevision).getKey())) {
+            throw new AbortException("Library '" + name + "' has been modified in an untrusted revision");
+        }
+    }
+}

src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-excludedVersionsStr.html

@@ -1,3 +1,3 @@
 <div>
-    Space separated list of versions excluded from caching. Ex: "master release10 release9"
+    Space separated list of versions to exclude from caching via substring search using .contains() method. Ex: "release/ master release10 release9".
 </div>

src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java

@@ -55,6 +55,7 @@ public class LibraryCachingConfigurationTest {
     private LibraryCachingConfiguration nullVersionConfig;
     private LibraryCachingConfiguration oneVersionConfig;
     private LibraryCachingConfiguration multiVersionConfig;
+    private LibraryCachingConfiguration substringVersionConfig;
 
     private static int REFRESH_TIME_MINUTES = 23;
     private static int NO_REFRESH_TIME_MINUTES = 0;
@@ -66,18 +67,25 @@ public class LibraryCachingConfigurationTest {
     private static String MULTIPLE_EXCLUDED_VERSIONS_2 = "branch-2";
     private static String MULTIPLE_EXCLUDED_VERSIONS_3 = "branch-3";
 
+    private static String SUBSTRING_EXCLUDED_VERSIONS_1 = "feature/test-substring-exclude";
+    private static String SUBSTRING_EXCLUDED_VERSIONS_2 = "test-other-substring-exclude";
+
     private static String MULTIPLE_EXCLUDED_VERSIONS =
         MULTIPLE_EXCLUDED_VERSIONS_1 + " " +
         MULTIPLE_EXCLUDED_VERSIONS_2 + " " +
         MULTIPLE_EXCLUDED_VERSIONS_3;
 
+    private static String SUBSTRING_EXCLUDED_VERSIONS =
+        "feature/ other-substring";
+
     private static String NEVER_EXCLUDED_VERSION = "never-excluded-version";
 
     @Before
     public void createCachingConfiguration() {
         nullVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, NULL_EXCLUDED_VERSION);
         oneVersionConfig = new LibraryCachingConfiguration(NO_REFRESH_TIME_MINUTES, ONE_EXCLUDED_VERSION);
         multiVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, MULTIPLE_EXCLUDED_VERSIONS);
+        substringVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, SUBSTRING_EXCLUDED_VERSIONS);
     }
 
     @Issue("JENKINS-66045") // NPE getting excluded versions
@@ -114,6 +122,7 @@ public void getExcludedVersionsStr() {
         assertThat(nullVersionConfig.getExcludedVersionsStr(), is(NULL_EXCLUDED_VERSION));
         assertThat(oneVersionConfig.getExcludedVersionsStr(), is(ONE_EXCLUDED_VERSION));
         assertThat(multiVersionConfig.getExcludedVersionsStr(), is(MULTIPLE_EXCLUDED_VERSIONS));
+        assertThat(substringVersionConfig.getExcludedVersionsStr(), is(SUBSTRING_EXCLUDED_VERSIONS));
     }
 
     @Test
@@ -128,17 +137,22 @@ public void isExcluded() {
         assertTrue(multiVersionConfig.isExcluded(MULTIPLE_EXCLUDED_VERSIONS_2));
         assertTrue(multiVersionConfig.isExcluded(MULTIPLE_EXCLUDED_VERSIONS_3));
 
+        assertTrue(substringVersionConfig.isExcluded(SUBSTRING_EXCLUDED_VERSIONS_1));
+        assertTrue(substringVersionConfig.isExcluded(SUBSTRING_EXCLUDED_VERSIONS_2));
+
         assertFalse(nullVersionConfig.isExcluded(NEVER_EXCLUDED_VERSION));
         assertFalse(oneVersionConfig.isExcluded(NEVER_EXCLUDED_VERSION));
         assertFalse(multiVersionConfig.isExcluded(NEVER_EXCLUDED_VERSION));
 
         assertFalse(nullVersionConfig.isExcluded(""));
         assertFalse(oneVersionConfig.isExcluded(""));
         assertFalse(multiVersionConfig.isExcluded(""));
+        assertFalse(substringVersionConfig.isExcluded(""));
 
         assertFalse(nullVersionConfig.isExcluded(null));
         assertFalse(oneVersionConfig.isExcluded(null));
         assertFalse(multiVersionConfig.isExcluded(null));
+        assertFalse(substringVersionConfig.isExcluded(null));
     }
 
     @Test

src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryMemoryTest.java

@@ -82,7 +82,7 @@ public static void register(Object o) {
             clearInvocationCaches.invoke(metaClass);
         }
         for (WeakReference<ClassLoader> loaderRef : LOADERS) {
-            MemoryAssert.assertGC(loaderRef, false);
+            MemoryAssert.assertGC(loaderRef);
         }
     }