Merge pull request #16 from AnishaGharat/feature/include-versions-caching

[JENKINS-69135] Add a "Versions to include" field to the Global Library Cache feature

Author: rsandell

src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryAdder.java

@@ -52,6 +52,7 @@
 import edu.umd.cs.findbugs.annotations.NonNull;
 import java.util.regex.Pattern;
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
 import org.jenkinsci.plugins.workflow.cps.CpsFlowExecution;
 import org.jenkinsci.plugins.workflow.cps.GlobalVariable;
 import org.jenkinsci.plugins.workflow.cps.GlobalVariableSet;
@@ -210,7 +211,10 @@ static List<URL> retrieve(@NonNull LibraryRecord record, @NonNull LibraryRetriev
             shouldCache = false;
         }
 
-        if(shouldCache) {
+        //If the included versions is blank/null, cache irrespective
+        //else check if that version is included and then cache only that version
+
+        if((shouldCache && cachingConfiguration.isIncluded(version)) || (shouldCache && StringUtils.isBlank(cachingConfiguration.getIncludedVersionsStr()))) {
             retrieveLock.readLock().lockInterruptibly();
             try {
                 CacheStatus cacheStatus = getCacheStatus(cachingConfiguration, versionCacheDir);
@@ -220,8 +224,8 @@ static List<URL> retrieve(@NonNull LibraryRecord record, @NonNull LibraryRetriev
                     try {
                       boolean retrieve = false;
                       switch (getCacheStatus(cachingConfiguration, versionCacheDir)) {
-                          case VALID: 
-                              listener.getLogger().println("Library " + name + "@" + version + " is cached. Copying from home."); 
+                          case VALID:
+                              listener.getLogger().println("Library " + name + "@" + version + " is cached. Copying from home.");
                                 break;
                           case DOES_NOT_EXIST:
                               retrieve = true;
@@ -236,20 +240,20 @@ static List<URL> retrieve(@NonNull LibraryRecord record, @NonNull LibraryRetriev
                                 retrieve = true;
                                 break;
                       }
-                            
+
                         if (retrieve) {
-                            listener.getLogger().println("Caching library " + name + "@" + version);                            
+                            listener.getLogger().println("Caching library " + name + "@" + version);
                             versionCacheDir.mkdirs();
                             retriever.retrieve(name, version, changelog, versionCacheDir, run, listener);
                         }
                         retrieveLock.readLock().lock();
                     } finally {
-                        retrieveLock.writeLock().unlock(); 
+                        retrieveLock.writeLock().unlock();
                     }
                 } else {
-                    listener.getLogger().println("Library " + name + "@" + version + " is cached. Copying from home.");  
+                    listener.getLogger().println("Library " + name + "@" + version + " is cached. Copying from home.");
                 }
-  
+
                 lastReadFile.touch(System.currentTimeMillis());
                 versionCacheDir.withSuffix("-name.txt").write(name, "UTF-8");
                 versionCacheDir.copyRecursiveTo(libDir);

src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java

@@ -2,11 +2,15 @@
 
 import hudson.Extension;
 import hudson.FilePath;
+import hudson.RestrictedSince;
 import hudson.model.AbstractDescribableImpl;
 import hudson.model.Descriptor;
 import hudson.util.FormValidation;
 import jenkins.model.Jenkins;
+import org.kohsuke.accmod.Restricted;
+import org.kohsuke.accmod.restrictions.NoExternalUse;
 import org.kohsuke.stapler.DataBoundConstructor;
+import org.kohsuke.stapler.DataBoundSetter;
 import org.kohsuke.stapler.QueryParameter;
 
 import java.io.IOException;
@@ -29,6 +33,7 @@ public final class LibraryCachingConfiguration extends AbstractDescribableImpl<L
 
     private int refreshTimeMinutes;
     private String excludedVersionsStr;
+    private String includedVersionsStr;
 
     private static final String VERSIONS_SEPARATOR = " ";
     public static final String GLOBAL_LIBRARIES_DIR = "global-libraries-cache";
@@ -37,6 +42,17 @@ public final class LibraryCachingConfiguration extends AbstractDescribableImpl<L
     @DataBoundConstructor public LibraryCachingConfiguration(int refreshTimeMinutes, String excludedVersionsStr) {
         this.refreshTimeMinutes = refreshTimeMinutes;
         this.excludedVersionsStr = excludedVersionsStr;
+        this.includedVersionsStr = "";
+    }
+
+    /*
+     * Visible for testing ...
+     */
+    @Restricted(NoExternalUse.class)
+    LibraryCachingConfiguration(int refreshTimeMinutes, String excludedVersionsStr, String includedVersionsStr) {
+        this.refreshTimeMinutes = refreshTimeMinutes;
+        this.excludedVersionsStr = excludedVersionsStr;
+        this.includedVersionsStr = includedVersionsStr;
     }
 
     public int getRefreshTimeMinutes() {
@@ -54,6 +70,17 @@ public Boolean isRefreshEnabled() {
     public String getExcludedVersionsStr() {
         return excludedVersionsStr;
     }
+    public String getIncludedVersionsStr() {
+        if(StringUtils.isBlank(includedVersionsStr)){
+            return null;
+        }
+            return includedVersionsStr;
+    }
+
+    @DataBoundSetter
+    public void setIncludedVersionsStr(String includedVersionsStr) {
+        this.includedVersionsStr = includedVersionsStr;
+    }
 
     private List<String> getExcludedVersions() {
         if (excludedVersionsStr == null) {
@@ -62,6 +89,13 @@ private List<String> getExcludedVersions() {
         return Arrays.asList(excludedVersionsStr.split(VERSIONS_SEPARATOR));
     }
 
+    private List<String> getIncludedVersions() {
+        if (includedVersionsStr == null) {
+            return Collections.emptyList();
+        }
+        return Arrays.asList(includedVersionsStr.split(VERSIONS_SEPARATOR));
+    }
+
     public Boolean isExcluded(String version) {
         // exit early if the version passed in is null or empty
         if (StringUtils.isBlank(version)) {
@@ -78,6 +112,22 @@ public Boolean isExcluded(String version) {
         return false;
     }
 
+    public Boolean isIncluded(String version) {
+        // exit early if the version passed in is null or empty
+        if (StringUtils.isBlank(version)) {
+            return false;
+        }
+        for (String it : getIncludedVersions()) {
+            // works on empty or null included versions
+            // and if the version contains the inclusion thus it can be
+            // anywhere in the string.
+            if (StringUtils.isNotBlank(it) && version.contains(it)){
+                return true;
+            }
+        }
+        return false;
+    }
+
     @Override public String toString() {
         return "LibraryCachingConfiguration{refreshTimeMinutes=" + refreshTimeMinutes + ", excludedVersions="
                 + excludedVersionsStr + '}';
@@ -129,6 +179,5 @@ public FormValidation doClearCache(@QueryParameter String name, @QueryParameter
             }
             return FormValidation.ok("The cache dir was deleted successfully.");
         }
-
     }
 }

src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/config.jelly

@@ -31,6 +31,9 @@ THE SOFTWARE.
     <f:entry title="${%Versions to exclude}" field="excludedVersionsStr">
         <f:textbox />
     </f:entry>
+     <f:entry title="${%Versions to include}" field="includedVersionsStr">
+        <f:textbox />
+     </f:entry>
     <j:if test="${h.hasPermission(app.ADMINISTER)}">
         <f:entry title="${%Force clear cache}" field="forceDelete">
           <f:checkbox/>

src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-includedVersionsStr.html

@@ -0,0 +1,6 @@
+<div>
+    Space separated list of versions to include to allow caching via substring search using .contains() method. Ex: "release/ master".
+</div>
+<div>
+    Note: Excluded versions will always take precedence over included versions
+</div>

src/test/java/org/jenkinsci/plugins/workflow/libs/GlobalLibrariesTest.java

@@ -57,6 +57,8 @@ public class GlobalLibrariesTest {
         assertEquals(Collections.emptyList(), gl.getLibraries());
         LibraryConfiguration foo = new LibraryConfiguration("foo", new SCMSourceRetriever(new SubversionSCMSource("foo", "https://phony.jenkins.io/foo/")));
         LibraryConfiguration bar = new LibraryConfiguration("bar", new SCMSourceRetriever(new GitSCMSource(null, "https://phony.jenkins.io/bar.git", "", "origin", "+refs/heads/*:refs/remotes/origin/*", "*", "", true)));
+        LibraryCachingConfiguration cachingConfiguration = new LibraryCachingConfiguration(120, "develop", "master stable");
+        foo.setCachingConfiguration(cachingConfiguration);
         bar.setDefaultVersion("master");
         bar.setImplicit(true);
         bar.setAllowVersionOverride(false);
@@ -72,6 +74,14 @@ public class GlobalLibrariesTest {
         r.assertEqualDataBoundBeans(Arrays.asList(foo, bar), libs);
         libs = gl.getLibraries();
         r.assertEqualDataBoundBeans(Arrays.asList(foo, bar), libs);
+        boolean noFoo = true;
+        for (LibraryConfiguration lib : libs) {
+            if ("foo".equals(lib.getName())) {
+                noFoo = false;
+                r.assertEqualDataBoundBeans(lib.getCachingConfiguration(), cachingConfiguration);
+            }
+        }
+        assertFalse("Missing a library called foo (should not happen)", noFoo);
     }
 
     @Issue("SECURITY-1422")

src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java

@@ -61,31 +61,57 @@ public class LibraryCachingConfigurationTest {
     private static int NO_REFRESH_TIME_MINUTES = 0;
 
     private static String NULL_EXCLUDED_VERSION = null;
+    private static String NULL_INCLUDED_VERSION = null;
+
     private static String ONE_EXCLUDED_VERSION = "branch-1";
 
+    private static String ONE_INCLUDED_VERSION = "branch-1i";
+
+
     private static String MULTIPLE_EXCLUDED_VERSIONS_1 = "main";
+
+    private static String MULTIPLE_INCLUDED_VERSIONS_1 = "master";
+
     private static String MULTIPLE_EXCLUDED_VERSIONS_2 = "branch-2";
+
+    private static String MULTIPLE_INCLUDED_VERSIONS_2 = "branch-2i";
+
     private static String MULTIPLE_EXCLUDED_VERSIONS_3 = "branch-3";
+    private static String MULTIPLE_INCLUDED_VERSIONS_3 = "branch-3i";
+
 
     private static String SUBSTRING_EXCLUDED_VERSIONS_1 = "feature/test-substring-exclude";
+
+    private static String SUBSTRING_INCLUDED_VERSIONS_1 = "feature_include/test-substring";
+
     private static String SUBSTRING_EXCLUDED_VERSIONS_2 = "test-other-substring-exclude";
+    private static String SUBSTRING_INCLUDED_VERSIONS_2 = "test-other-substring-include";
+
 
     private static String MULTIPLE_EXCLUDED_VERSIONS =
         MULTIPLE_EXCLUDED_VERSIONS_1 + " " +
         MULTIPLE_EXCLUDED_VERSIONS_2 + " " +
         MULTIPLE_EXCLUDED_VERSIONS_3;
 
+    private static String MULTIPLE_INCLUDED_VERSIONS =
+            MULTIPLE_INCLUDED_VERSIONS_1 + " " +
+            MULTIPLE_INCLUDED_VERSIONS_2 + " " +
+            MULTIPLE_INCLUDED_VERSIONS_3;
+
     private static String SUBSTRING_EXCLUDED_VERSIONS =
         "feature/ other-substring";
 
+    private static String SUBSTRING_INCLUDED_VERSIONS =
+            "feature_include/ other-substring";
+
     private static String NEVER_EXCLUDED_VERSION = "never-excluded-version";
 
     @Before
     public void createCachingConfiguration() {
-        nullVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, NULL_EXCLUDED_VERSION);
-        oneVersionConfig = new LibraryCachingConfiguration(NO_REFRESH_TIME_MINUTES, ONE_EXCLUDED_VERSION);
-        multiVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, MULTIPLE_EXCLUDED_VERSIONS);
-        substringVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, SUBSTRING_EXCLUDED_VERSIONS);
+        nullVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, NULL_EXCLUDED_VERSION, NULL_INCLUDED_VERSION);
+        oneVersionConfig = new LibraryCachingConfiguration(NO_REFRESH_TIME_MINUTES, ONE_EXCLUDED_VERSION, ONE_INCLUDED_VERSION);
+        multiVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, MULTIPLE_EXCLUDED_VERSIONS, MULTIPLE_INCLUDED_VERSIONS);
+        substringVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, SUBSTRING_EXCLUDED_VERSIONS, SUBSTRING_INCLUDED_VERSIONS);
     }
 
     @Issue("JENKINS-66045") // NPE getting excluded versions
@@ -125,6 +151,15 @@ public void getExcludedVersionsStr() {
         assertThat(substringVersionConfig.getExcludedVersionsStr(), is(SUBSTRING_EXCLUDED_VERSIONS));
     }
 
+    @Test
+    @WithoutJenkins
+    public void getIncludedVersionsStr() {
+        assertThat(nullVersionConfig.getIncludedVersionsStr(), is(NULL_INCLUDED_VERSION));
+        assertThat(oneVersionConfig.getIncludedVersionsStr(), is(ONE_INCLUDED_VERSION));
+        assertThat(multiVersionConfig.getIncludedVersionsStr(), is(MULTIPLE_INCLUDED_VERSIONS));
+        assertThat(substringVersionConfig.getIncludedVersionsStr(), is(SUBSTRING_INCLUDED_VERSIONS));
+    }
+
     @Test
     @WithoutJenkins
     public void isExcluded() {
@@ -155,6 +190,24 @@ public void isExcluded() {
         assertFalse(substringVersionConfig.isExcluded(null));
     }
 
+    @Issue("JENKINS-69135") //"Versions to include" feature for caching
+    @Test
+    @WithoutJenkins
+    public void isIncluded() {
+        assertFalse(nullVersionConfig.isIncluded(NULL_INCLUDED_VERSION));
+        assertFalse(nullVersionConfig.isIncluded(""));
+
+        assertTrue(oneVersionConfig.isIncluded(ONE_INCLUDED_VERSION));
+
+        assertTrue(multiVersionConfig.isIncluded(MULTIPLE_INCLUDED_VERSIONS_1));
+        assertTrue(multiVersionConfig.isIncluded(MULTIPLE_INCLUDED_VERSIONS_2));
+        assertTrue(multiVersionConfig.isIncluded(MULTIPLE_INCLUDED_VERSIONS_3));
+
+        assertTrue(substringVersionConfig.isIncluded(SUBSTRING_INCLUDED_VERSIONS_1));
+        assertTrue(substringVersionConfig.isIncluded(SUBSTRING_INCLUDED_VERSIONS_2));
+
+    }
+
     @Test
     public void clearCache() throws Exception {
         sampleRepo.init();
@@ -182,4 +235,72 @@ public void clearCache() throws Exception {
         assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile()));
     }
 
+    //Test similar substrings in "Versions to include" & "Versions to exclude"
+    //Exclusion takes precedence
+    @Issue("JENKINS-69135") //"Versions to include" feature for caching
+    @Test
+    public void clearCacheConflict() throws Exception {
+        sampleRepo.init();
+        sampleRepo.write("vars/foo.groovy", "def call() { echo 'foo' }");
+        sampleRepo.git("add", "vars");
+        sampleRepo.git("commit", "--message=init");
+        LibraryConfiguration config = new LibraryConfiguration("library",
+                new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true)));
+        config.setDefaultVersion("master");
+        config.setImplicit(true);
+        // Same version specified in both include and exclude version
+        //Exclude takes precedence
+        config.setCachingConfiguration(new LibraryCachingConfiguration(30, "master", "master"));
+        GlobalLibraries.get().getLibraries().add(config);
+        // Run build and check that cache gets created.
+        WorkflowJob p = r.createProject(WorkflowJob.class);
+        p.setDefinition(new CpsFlowDefinition("foo()", true));
+        WorkflowRun b = r.buildAndAssertSuccess(p);
+        LibrariesAction action = b.getAction(LibrariesAction.class);
+        LibraryRecord record = action.getLibraries().get(0);
+        FilePath cache = LibraryCachingConfiguration.getGlobalLibrariesCacheDir().child(record.getDirectoryName());
+        // Cache should not get created since the version is included in "Versions to exclude"
+        assertThat(new File(cache.getRemote()), not(anExistingDirectory()));
+        assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile()));
+    }
+
+    @Issue("JENKINS-69135") //"Versions to include" feature for caching
+    @Test
+    public void clearCacheIncludedVersion() throws Exception {
+        sampleRepo.init();
+        sampleRepo.write("vars/foo.groovy", "def call() { echo 'foo' }");
+        sampleRepo.git("add", "vars");
+        sampleRepo.git("commit", "--message=init");
+        sampleRepo.git("branch", "test/include");
+        LibraryConfiguration config = new LibraryConfiguration("library",
+                new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true)));
+        config.setDefaultVersion("master");
+        config.setAllowVersionOverride(true);
+        config.setImplicit(false);
+        config.setCachingConfiguration(new LibraryCachingConfiguration(30, "", "test/include"));
+        GlobalLibraries.get().getLibraries().add(config);
+        // Run build and check that cache gets created.
+        WorkflowJob p = r.createProject(WorkflowJob.class);
+        p.setDefinition(new CpsFlowDefinition("library identifier: 'library', changelog:false\n\nfoo()", true));
+        WorkflowRun b = r.buildAndAssertSuccess(p);
+        WorkflowJob p2 = r.createProject(WorkflowJob.class);
+        p2.setDefinition(new CpsFlowDefinition("library identifier: 'library@test/include', changelog:false\n\nfoo()", true));
+        WorkflowRun b2 = r.buildAndAssertSuccess(p2);
+        LibrariesAction action = b.getAction(LibrariesAction.class);
+        LibraryRecord record = action.getLibraries().get(0);
+        LibrariesAction action2 = b2.getAction(LibrariesAction.class);
+        LibraryRecord record2 = action2.getLibraries().get(0);
+        FilePath cache = LibraryCachingConfiguration.getGlobalLibrariesCacheDir().child(record.getDirectoryName());
+        FilePath cache2 = LibraryCachingConfiguration.getGlobalLibrariesCacheDir().child(record2.getDirectoryName());
+        assertThat(new File(cache.getRemote()), not(anExistingDirectory()));
+        assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile()));
+        assertThat(new File(cache2.getRemote()), anExistingDirectory());
+        assertThat(new File(cache2.withSuffix("-name.txt").getRemote()), anExistingFile());
+        // Clears cache for the entire library, until the "Delete specific cache version" feature in merged
+        // Clear the cache. TODO: Would be more realistic to set up security and use WebClient.
+        ExtensionList.lookupSingleton(LibraryCachingConfiguration.DescriptorImpl.class).doClearCache("library", false);
+        assertThat(new File(cache2.getRemote()), not(anExistingDirectory()));
+        assertThat(new File(cache2.withSuffix("-name.txt").getRemote()), not(anExistingFile()));
+    }
+
 }