diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryAdder.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryAdder.java index 9ff64777..a57ba1e6 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryAdder.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryAdder.java @@ -52,6 +52,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; import java.util.regex.Pattern; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; import org.jenkinsci.plugins.workflow.cps.CpsFlowExecution; import org.jenkinsci.plugins.workflow.cps.GlobalVariable; import org.jenkinsci.plugins.workflow.cps.GlobalVariableSet; @@ -210,7 +211,10 @@ static List retrieve(@NonNull LibraryRecord record, @NonNull LibraryRetriev shouldCache = false; } - if(shouldCache) { + //If the included versions is blank/null, cache irrespective + //else check if that version is included and then cache only that version + + if((shouldCache && cachingConfiguration.isIncluded(version)) || (shouldCache && StringUtils.isBlank(cachingConfiguration.getIncludedVersionsStr()))) { retrieveLock.readLock().lockInterruptibly(); try { CacheStatus cacheStatus = getCacheStatus(cachingConfiguration, versionCacheDir); @@ -220,8 +224,8 @@ static List retrieve(@NonNull LibraryRecord record, @NonNull LibraryRetriev try { boolean retrieve = false; switch (getCacheStatus(cachingConfiguration, versionCacheDir)) { - case VALID: - listener.getLogger().println("Library " + name + "@" + version + " is cached. Copying from home."); + case VALID: + listener.getLogger().println("Library " + name + "@" + version + " is cached. Copying from home."); break; case DOES_NOT_EXIST: retrieve = true; @@ -236,20 +240,20 @@ static List retrieve(@NonNull LibraryRecord record, @NonNull LibraryRetriev retrieve = true; break; } - + if (retrieve) { - listener.getLogger().println("Caching library " + name + "@" + version); + listener.getLogger().println("Caching library " + name + "@" + version); versionCacheDir.mkdirs(); retriever.retrieve(name, version, changelog, versionCacheDir, run, listener); } retrieveLock.readLock().lock(); } finally { - retrieveLock.writeLock().unlock(); + retrieveLock.writeLock().unlock(); } } else { - listener.getLogger().println("Library " + name + "@" + version + " is cached. Copying from home."); + listener.getLogger().println("Library " + name + "@" + version + " is cached. Copying from home."); } - + lastReadFile.touch(System.currentTimeMillis()); versionCacheDir.withSuffix("-name.txt").write(name, "UTF-8"); versionCacheDir.copyRecursiveTo(libDir); diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java index afc3dfad..3234edb8 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java @@ -2,11 +2,15 @@ import hudson.Extension; import hudson.FilePath; +import hudson.RestrictedSince; import hudson.model.AbstractDescribableImpl; import hudson.model.Descriptor; import hudson.util.FormValidation; import jenkins.model.Jenkins; +import org.kohsuke.accmod.Restricted; +import org.kohsuke.accmod.restrictions.NoExternalUse; import org.kohsuke.stapler.DataBoundConstructor; +import org.kohsuke.stapler.DataBoundSetter; import org.kohsuke.stapler.QueryParameter; import java.io.IOException; @@ -29,6 +33,7 @@ public final class LibraryCachingConfiguration extends AbstractDescribableImpl getExcludedVersions() { if (excludedVersionsStr == null) { @@ -62,6 +89,13 @@ private List getExcludedVersions() { return Arrays.asList(excludedVersionsStr.split(VERSIONS_SEPARATOR)); } + private List getIncludedVersions() { + if (includedVersionsStr == null) { + return Collections.emptyList(); + } + return Arrays.asList(includedVersionsStr.split(VERSIONS_SEPARATOR)); + } + public Boolean isExcluded(String version) { // exit early if the version passed in is null or empty if (StringUtils.isBlank(version)) { @@ -78,6 +112,22 @@ public Boolean isExcluded(String version) { return false; } + public Boolean isIncluded(String version) { + // exit early if the version passed in is null or empty + if (StringUtils.isBlank(version)) { + return false; + } + for (String it : getIncludedVersions()) { + // works on empty or null included versions + // and if the version contains the inclusion thus it can be + // anywhere in the string. + if (StringUtils.isNotBlank(it) && version.contains(it)){ + return true; + } + } + return false; + } + @Override public String toString() { return "LibraryCachingConfiguration{refreshTimeMinutes=" + refreshTimeMinutes + ", excludedVersions=" + excludedVersionsStr + '}'; @@ -129,6 +179,5 @@ public FormValidation doClearCache(@QueryParameter String name, @QueryParameter } return FormValidation.ok("The cache dir was deleted successfully."); } - } } \ No newline at end of file diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/config.jelly b/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/config.jelly index 584d024a..8219f0c7 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/config.jelly +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/config.jelly @@ -31,6 +31,9 @@ THE SOFTWARE. + + + diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-includedVersionsStr.html b/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-includedVersionsStr.html new file mode 100644 index 00000000..27365936 --- /dev/null +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-includedVersionsStr.html @@ -0,0 +1,6 @@ +
+ Space separated list of versions to include to allow caching via substring search using .contains() method. Ex: "release/ master". +
+
+ Note: Excluded versions will always take precedence over included versions +
\ No newline at end of file diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/GlobalLibrariesTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/GlobalLibrariesTest.java index 61da2492..efadbe5d 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/GlobalLibrariesTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/GlobalLibrariesTest.java @@ -57,6 +57,8 @@ public class GlobalLibrariesTest { assertEquals(Collections.emptyList(), gl.getLibraries()); LibraryConfiguration foo = new LibraryConfiguration("foo", new SCMSourceRetriever(new SubversionSCMSource("foo", "https://phony.jenkins.io/foo/"))); LibraryConfiguration bar = new LibraryConfiguration("bar", new SCMSourceRetriever(new GitSCMSource(null, "https://phony.jenkins.io/bar.git", "", "origin", "+refs/heads/*:refs/remotes/origin/*", "*", "", true))); + LibraryCachingConfiguration cachingConfiguration = new LibraryCachingConfiguration(120, "develop", "master stable"); + foo.setCachingConfiguration(cachingConfiguration); bar.setDefaultVersion("master"); bar.setImplicit(true); bar.setAllowVersionOverride(false); @@ -72,6 +74,14 @@ public class GlobalLibrariesTest { r.assertEqualDataBoundBeans(Arrays.asList(foo, bar), libs); libs = gl.getLibraries(); r.assertEqualDataBoundBeans(Arrays.asList(foo, bar), libs); + boolean noFoo = true; + for (LibraryConfiguration lib : libs) { + if ("foo".equals(lib.getName())) { + noFoo = false; + r.assertEqualDataBoundBeans(lib.getCachingConfiguration(), cachingConfiguration); + } + } + assertFalse("Missing a library called foo (should not happen)", noFoo); } @Issue("SECURITY-1422") diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java index b2a0a55f..a3689e27 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java @@ -61,31 +61,57 @@ public class LibraryCachingConfigurationTest { private static int NO_REFRESH_TIME_MINUTES = 0; private static String NULL_EXCLUDED_VERSION = null; + private static String NULL_INCLUDED_VERSION = null; + private static String ONE_EXCLUDED_VERSION = "branch-1"; + private static String ONE_INCLUDED_VERSION = "branch-1i"; + + private static String MULTIPLE_EXCLUDED_VERSIONS_1 = "main"; + + private static String MULTIPLE_INCLUDED_VERSIONS_1 = "master"; + private static String MULTIPLE_EXCLUDED_VERSIONS_2 = "branch-2"; + + private static String MULTIPLE_INCLUDED_VERSIONS_2 = "branch-2i"; + private static String MULTIPLE_EXCLUDED_VERSIONS_3 = "branch-3"; + private static String MULTIPLE_INCLUDED_VERSIONS_3 = "branch-3i"; + private static String SUBSTRING_EXCLUDED_VERSIONS_1 = "feature/test-substring-exclude"; + + private static String SUBSTRING_INCLUDED_VERSIONS_1 = "feature_include/test-substring"; + private static String SUBSTRING_EXCLUDED_VERSIONS_2 = "test-other-substring-exclude"; + private static String SUBSTRING_INCLUDED_VERSIONS_2 = "test-other-substring-include"; + private static String MULTIPLE_EXCLUDED_VERSIONS = MULTIPLE_EXCLUDED_VERSIONS_1 + " " + MULTIPLE_EXCLUDED_VERSIONS_2 + " " + MULTIPLE_EXCLUDED_VERSIONS_3; + private static String MULTIPLE_INCLUDED_VERSIONS = + MULTIPLE_INCLUDED_VERSIONS_1 + " " + + MULTIPLE_INCLUDED_VERSIONS_2 + " " + + MULTIPLE_INCLUDED_VERSIONS_3; + private static String SUBSTRING_EXCLUDED_VERSIONS = "feature/ other-substring"; + private static String SUBSTRING_INCLUDED_VERSIONS = + "feature_include/ other-substring"; + private static String NEVER_EXCLUDED_VERSION = "never-excluded-version"; @Before public void createCachingConfiguration() { - nullVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, NULL_EXCLUDED_VERSION); - oneVersionConfig = new LibraryCachingConfiguration(NO_REFRESH_TIME_MINUTES, ONE_EXCLUDED_VERSION); - multiVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, MULTIPLE_EXCLUDED_VERSIONS); - substringVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, SUBSTRING_EXCLUDED_VERSIONS); + nullVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, NULL_EXCLUDED_VERSION, NULL_INCLUDED_VERSION); + oneVersionConfig = new LibraryCachingConfiguration(NO_REFRESH_TIME_MINUTES, ONE_EXCLUDED_VERSION, ONE_INCLUDED_VERSION); + multiVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, MULTIPLE_EXCLUDED_VERSIONS, MULTIPLE_INCLUDED_VERSIONS); + substringVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, SUBSTRING_EXCLUDED_VERSIONS, SUBSTRING_INCLUDED_VERSIONS); } @Issue("JENKINS-66045") // NPE getting excluded versions @@ -125,6 +151,15 @@ public void getExcludedVersionsStr() { assertThat(substringVersionConfig.getExcludedVersionsStr(), is(SUBSTRING_EXCLUDED_VERSIONS)); } + @Test + @WithoutJenkins + public void getIncludedVersionsStr() { + assertThat(nullVersionConfig.getIncludedVersionsStr(), is(NULL_INCLUDED_VERSION)); + assertThat(oneVersionConfig.getIncludedVersionsStr(), is(ONE_INCLUDED_VERSION)); + assertThat(multiVersionConfig.getIncludedVersionsStr(), is(MULTIPLE_INCLUDED_VERSIONS)); + assertThat(substringVersionConfig.getIncludedVersionsStr(), is(SUBSTRING_INCLUDED_VERSIONS)); + } + @Test @WithoutJenkins public void isExcluded() { @@ -155,6 +190,24 @@ public void isExcluded() { assertFalse(substringVersionConfig.isExcluded(null)); } + @Issue("JENKINS-69135") //"Versions to include" feature for caching + @Test + @WithoutJenkins + public void isIncluded() { + assertFalse(nullVersionConfig.isIncluded(NULL_INCLUDED_VERSION)); + assertFalse(nullVersionConfig.isIncluded("")); + + assertTrue(oneVersionConfig.isIncluded(ONE_INCLUDED_VERSION)); + + assertTrue(multiVersionConfig.isIncluded(MULTIPLE_INCLUDED_VERSIONS_1)); + assertTrue(multiVersionConfig.isIncluded(MULTIPLE_INCLUDED_VERSIONS_2)); + assertTrue(multiVersionConfig.isIncluded(MULTIPLE_INCLUDED_VERSIONS_3)); + + assertTrue(substringVersionConfig.isIncluded(SUBSTRING_INCLUDED_VERSIONS_1)); + assertTrue(substringVersionConfig.isIncluded(SUBSTRING_INCLUDED_VERSIONS_2)); + + } + @Test public void clearCache() throws Exception { sampleRepo.init(); @@ -182,4 +235,72 @@ public void clearCache() throws Exception { assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile())); } + //Test similar substrings in "Versions to include" & "Versions to exclude" + //Exclusion takes precedence + @Issue("JENKINS-69135") //"Versions to include" feature for caching + @Test + public void clearCacheConflict() throws Exception { + sampleRepo.init(); + sampleRepo.write("vars/foo.groovy", "def call() { echo 'foo' }"); + sampleRepo.git("add", "vars"); + sampleRepo.git("commit", "--message=init"); + LibraryConfiguration config = new LibraryConfiguration("library", + new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); + config.setDefaultVersion("master"); + config.setImplicit(true); + // Same version specified in both include and exclude version + //Exclude takes precedence + config.setCachingConfiguration(new LibraryCachingConfiguration(30, "master", "master")); + GlobalLibraries.get().getLibraries().add(config); + // Run build and check that cache gets created. + WorkflowJob p = r.createProject(WorkflowJob.class); + p.setDefinition(new CpsFlowDefinition("foo()", true)); + WorkflowRun b = r.buildAndAssertSuccess(p); + LibrariesAction action = b.getAction(LibrariesAction.class); + LibraryRecord record = action.getLibraries().get(0); + FilePath cache = LibraryCachingConfiguration.getGlobalLibrariesCacheDir().child(record.getDirectoryName()); + // Cache should not get created since the version is included in "Versions to exclude" + assertThat(new File(cache.getRemote()), not(anExistingDirectory())); + assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile())); + } + + @Issue("JENKINS-69135") //"Versions to include" feature for caching + @Test + public void clearCacheIncludedVersion() throws Exception { + sampleRepo.init(); + sampleRepo.write("vars/foo.groovy", "def call() { echo 'foo' }"); + sampleRepo.git("add", "vars"); + sampleRepo.git("commit", "--message=init"); + sampleRepo.git("branch", "test/include"); + LibraryConfiguration config = new LibraryConfiguration("library", + new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); + config.setDefaultVersion("master"); + config.setAllowVersionOverride(true); + config.setImplicit(false); + config.setCachingConfiguration(new LibraryCachingConfiguration(30, "", "test/include")); + GlobalLibraries.get().getLibraries().add(config); + // Run build and check that cache gets created. + WorkflowJob p = r.createProject(WorkflowJob.class); + p.setDefinition(new CpsFlowDefinition("library identifier: 'library', changelog:false\n\nfoo()", true)); + WorkflowRun b = r.buildAndAssertSuccess(p); + WorkflowJob p2 = r.createProject(WorkflowJob.class); + p2.setDefinition(new CpsFlowDefinition("library identifier: 'library@test/include', changelog:false\n\nfoo()", true)); + WorkflowRun b2 = r.buildAndAssertSuccess(p2); + LibrariesAction action = b.getAction(LibrariesAction.class); + LibraryRecord record = action.getLibraries().get(0); + LibrariesAction action2 = b2.getAction(LibrariesAction.class); + LibraryRecord record2 = action2.getLibraries().get(0); + FilePath cache = LibraryCachingConfiguration.getGlobalLibrariesCacheDir().child(record.getDirectoryName()); + FilePath cache2 = LibraryCachingConfiguration.getGlobalLibrariesCacheDir().child(record2.getDirectoryName()); + assertThat(new File(cache.getRemote()), not(anExistingDirectory())); + assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile())); + assertThat(new File(cache2.getRemote()), anExistingDirectory()); + assertThat(new File(cache2.withSuffix("-name.txt").getRemote()), anExistingFile()); + // Clears cache for the entire library, until the "Delete specific cache version" feature in merged + // Clear the cache. TODO: Would be more realistic to set up security and use WebClient. + ExtensionList.lookupSingleton(LibraryCachingConfiguration.DescriptorImpl.class).doClearCache("library", false); + assertThat(new File(cache2.getRemote()), not(anExistingDirectory())); + assertThat(new File(cache2.withSuffix("-name.txt").getRemote()), not(anExistingFile())); + } + }