From f2ae5b708bd2dec2e909221d5080b050105fd3ee Mon Sep 17 00:00:00 2001 From: "KINAXIS\\agharat" Date: Mon, 15 Aug 2022 15:40:27 -0400 Subject: [PATCH 1/8] Add include versions to enable caching --- .../plugins/workflow/libs/LibraryAdder.java | 2 +- .../libs/LibraryCachingConfiguration.java | 62 ++++++++++++- .../LibraryCachingConfiguration/config.jelly | 3 + .../help-includedVersionsStr.html | 6 ++ .../workflow/libs/LibraryAdderTest.java | 6 +- .../libs/LibraryCachingCleanupTest.java | 2 +- .../libs/LibraryCachingConfigurationTest.java | 87 +++++++++++++++++-- .../workflow/libs/ResourceStepTest.java | 6 +- 8 files changed, 160 insertions(+), 14 deletions(-) create mode 100644 src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-includedVersionsStr.html diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryAdder.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryAdder.java index 9ff64777..1b7b879a 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryAdder.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryAdder.java @@ -210,7 +210,7 @@ static List retrieve(@NonNull LibraryRecord record, @NonNull LibraryRetriev shouldCache = false; } - if(shouldCache) { + if(shouldCache && cachingConfiguration.isIncluded(version)) { retrieveLock.readLock().lockInterruptibly(); try { CacheStatus cacheStatus = getCacheStatus(cachingConfiguration, versionCacheDir); diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java index afc3dfad..944033a7 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java @@ -29,16 +29,19 @@ public final class LibraryCachingConfiguration extends AbstractDescribableImpl getExcludedVersions() { if (excludedVersionsStr == null) { @@ -62,6 +67,13 @@ private List getExcludedVersions() { return Arrays.asList(excludedVersionsStr.split(VERSIONS_SEPARATOR)); } + private List getIncludedVersions() { + if (includedVersionsStr == null) { + return Collections.emptyList(); + } + return Arrays.asList(includedVersionsStr.split(VERSIONS_SEPARATOR)); + } + public Boolean isExcluded(String version) { // exit early if the version passed in is null or empty if (StringUtils.isBlank(version)) { @@ -78,6 +90,22 @@ public Boolean isExcluded(String version) { return false; } + public Boolean isIncluded(String version) { + // exit early if the version passed in is null or empty + if (StringUtils.isBlank(version)) { + return false; + } + for (String it : getIncludedVersions()) { + // confirm that the excluded versions aren't null or empty + // and if the version contains the exclusion thus it can be + // anywhere in the string. + if (StringUtils.isNotBlank(it) && version.contains(it)){ + return true; + } + } + return false; + } + @Override public String toString() { return "LibraryCachingConfiguration{refreshTimeMinutes=" + refreshTimeMinutes + ", excludedVersions=" + excludedVersionsStr + '}'; @@ -129,6 +157,38 @@ public FormValidation doClearCache(@QueryParameter String name, @QueryParameter } return FormValidation.ok("The cache dir was deleted successfully."); } + } + /** + * Method can be called from an external source to delete cache of a particular version of the shared library + * Example: delete cache from through pipeline script once the build is successful + * @param jenkinsLibrary Name of the shared library + * @param version Name of the version to delete the cache + * @throws IOException + * @throws InterruptedException + */ + public static void deleteLibraryCacheForBranch(String jenkinsLibrary, String version) throws IOException, InterruptedException { + if (LibraryCachingConfiguration.getGlobalLibrariesCacheDir().exists()) { + for (FilePath libraryNamePath : LibraryCachingConfiguration.getGlobalLibrariesCacheDir().list("*-name.txt")) { + String cacheName; + try (InputStream stream = libraryNamePath.read()) { + cacheName = IOUtils.toString(stream, StandardCharsets.UTF_8); + + if (cacheName.equals(jenkinsLibrary)) { + FilePath libraryCachePath = LibraryCachingConfiguration.getGlobalLibrariesCacheDir() + .child(libraryNamePath.getName().replace("-name.txt", "")); + ReentrantReadWriteLock retrieveLock = LibraryAdder.getReadWriteLockFor(libraryCachePath.getName()); + if (retrieveLock.writeLock().tryLock(10, TimeUnit.SECONDS)) { + try { + libraryCachePath.deleteRecursive(); + libraryNamePath.delete(); + } finally { + retrieveLock.writeLock().unlock(); + } + } + } + } + } + } } } \ No newline at end of file diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/config.jelly b/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/config.jelly index 584d024a..8219f0c7 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/config.jelly +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/config.jelly @@ -31,6 +31,9 @@ THE SOFTWARE. + + + diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-includedVersionsStr.html b/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-includedVersionsStr.html new file mode 100644 index 00000000..27365936 --- /dev/null +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-includedVersionsStr.html @@ -0,0 +1,6 @@ +
+ Space separated list of versions to include to allow caching via substring search using .contains() method. Ex: "release/ master". +
+
+ Note: Excluded versions will always take precedence over included versions +
\ No newline at end of file diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryAdderTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryAdderTest.java index 2cd590b8..3ef953b8 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryAdderTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryAdderTest.java @@ -420,7 +420,7 @@ public class LibraryAdderTest { new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); globalLib.setDefaultVersion("master"); globalLib.setImplicit(true); - globalLib.setCachingConfiguration(new LibraryCachingConfiguration(60, "")); + globalLib.setCachingConfiguration(new LibraryCachingConfiguration(60, "", "")); GlobalLibraries.get().setLibraries(Collections.singletonList(globalLib)); // Create a folder library with the same name and which is also set up to enable caching. sampleRepo2.write("vars/folderLibVar.groovy", "def call() { jenkins.model.Jenkins.get().setSystemMessage('folder library') }"); @@ -430,7 +430,7 @@ public class LibraryAdderTest { new SCMSourceRetriever(new GitSCMSource(null, sampleRepo2.toString(), "", "*", "", true))); folderLib.setDefaultVersion("master"); folderLib.setImplicit(true); - folderLib.setCachingConfiguration(new LibraryCachingConfiguration(60, "")); + folderLib.setCachingConfiguration(new LibraryCachingConfiguration(60, "", "")); Folder f = r.jenkins.createProject(Folder.class, "folder1"); f.getProperties().add(new FolderLibraries(Collections.singletonList(folderLib))); // Create a job that uses the folder library, which will take precedence over the global library, since they have the same name. @@ -493,7 +493,7 @@ public void parallelBuildsDontInterfereWithExpiredCache() throws Throwable { new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); config.setDefaultVersion("master"); config.setImplicit(true); - config.setCachingConfiguration(new LibraryCachingConfiguration(30, null)); + config.setCachingConfiguration(new LibraryCachingConfiguration(30, null,null)); GlobalLibraries.get().getLibraries().add(config); WorkflowJob p1 = r.createProject(WorkflowJob.class); WorkflowJob p2 = r.createProject(WorkflowJob.class); diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanupTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanupTest.java index ff30c86e..9a66cb71 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanupTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanupTest.java @@ -59,7 +59,7 @@ public void smokes() throws Throwable { new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); config.setDefaultVersion("master"); config.setImplicit(true); - config.setCachingConfiguration(new LibraryCachingConfiguration(30, null)); + config.setCachingConfiguration(new LibraryCachingConfiguration(30, null, null)); GlobalLibraries.get().getLibraries().add(config); // Run build and check that cache gets created. WorkflowJob p = r.createProject(WorkflowJob.class); diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java index b2a0a55f..95fe8d42 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java @@ -61,31 +61,57 @@ public class LibraryCachingConfigurationTest { private static int NO_REFRESH_TIME_MINUTES = 0; private static String NULL_EXCLUDED_VERSION = null; + private static String NULL_INCLUDED_VERSION = null; + private static String ONE_EXCLUDED_VERSION = "branch-1"; + private static String ONE_INCLUDED_VERSION = "branch-1i"; + + private static String MULTIPLE_EXCLUDED_VERSIONS_1 = "main"; + + private static String MULTIPLE_INCLUDED_VERSIONS_1 = "master"; + private static String MULTIPLE_EXCLUDED_VERSIONS_2 = "branch-2"; + + private static String MULTIPLE_INCLUDED_VERSIONS_2 = "branch-2i"; + private static String MULTIPLE_EXCLUDED_VERSIONS_3 = "branch-3"; + private static String MULTIPLE_INCLUDED_VERSIONS_3 = "branch-3i"; + private static String SUBSTRING_EXCLUDED_VERSIONS_1 = "feature/test-substring-exclude"; + + private static String SUBSTRING_INCLUDED_VERSIONS_1 = "feature_include/test-substring"; + private static String SUBSTRING_EXCLUDED_VERSIONS_2 = "test-other-substring-exclude"; + private static String SUBSTRING_INCLUDED_VERSIONS_2 = "test-other-substring-include"; + private static String MULTIPLE_EXCLUDED_VERSIONS = MULTIPLE_EXCLUDED_VERSIONS_1 + " " + MULTIPLE_EXCLUDED_VERSIONS_2 + " " + MULTIPLE_EXCLUDED_VERSIONS_3; + private static String MULTIPLE_INCLUDED_VERSIONS = + MULTIPLE_INCLUDED_VERSIONS_1 + " " + + MULTIPLE_INCLUDED_VERSIONS_2 + " " + + MULTIPLE_INCLUDED_VERSIONS_3; + private static String SUBSTRING_EXCLUDED_VERSIONS = "feature/ other-substring"; + private static String SUBSTRING_INCLUDED_VERSIONS = + "feature_include/ other-substring"; + private static String NEVER_EXCLUDED_VERSION = "never-excluded-version"; @Before public void createCachingConfiguration() { - nullVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, NULL_EXCLUDED_VERSION); - oneVersionConfig = new LibraryCachingConfiguration(NO_REFRESH_TIME_MINUTES, ONE_EXCLUDED_VERSION); - multiVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, MULTIPLE_EXCLUDED_VERSIONS); - substringVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, SUBSTRING_EXCLUDED_VERSIONS); + nullVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, NULL_EXCLUDED_VERSION, NULL_INCLUDED_VERSION); + oneVersionConfig = new LibraryCachingConfiguration(NO_REFRESH_TIME_MINUTES, ONE_EXCLUDED_VERSION, ONE_INCLUDED_VERSION); + multiVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, MULTIPLE_EXCLUDED_VERSIONS, MULTIPLE_INCLUDED_VERSIONS); + substringVersionConfig = new LibraryCachingConfiguration(REFRESH_TIME_MINUTES, SUBSTRING_EXCLUDED_VERSIONS, SUBSTRING_INCLUDED_VERSIONS); } @Issue("JENKINS-66045") // NPE getting excluded versions @@ -125,6 +151,15 @@ public void getExcludedVersionsStr() { assertThat(substringVersionConfig.getExcludedVersionsStr(), is(SUBSTRING_EXCLUDED_VERSIONS)); } + @Test + @WithoutJenkins + public void getIncludedVersionsStr() { + assertThat(nullVersionConfig.getIncludedVersionsStr(), is(NULL_INCLUDED_VERSION)); + assertThat(oneVersionConfig.getIncludedVersionsStr(), is(ONE_INCLUDED_VERSION)); + assertThat(multiVersionConfig.getIncludedVersionsStr(), is(MULTIPLE_INCLUDED_VERSIONS)); + assertThat(substringVersionConfig.getIncludedVersionsStr(), is(SUBSTRING_INCLUDED_VERSIONS)); + } + @Test @WithoutJenkins public void isExcluded() { @@ -155,6 +190,23 @@ public void isExcluded() { assertFalse(substringVersionConfig.isExcluded(null)); } + @Test + @WithoutJenkins + public void isIncluded() { + assertFalse(nullVersionConfig.isIncluded(NULL_INCLUDED_VERSION)); + assertFalse(nullVersionConfig.isIncluded("")); + + assertTrue(oneVersionConfig.isIncluded(ONE_INCLUDED_VERSION)); + + assertTrue(multiVersionConfig.isIncluded(MULTIPLE_INCLUDED_VERSIONS_1)); + assertTrue(multiVersionConfig.isIncluded(MULTIPLE_INCLUDED_VERSIONS_2)); + assertTrue(multiVersionConfig.isIncluded(MULTIPLE_INCLUDED_VERSIONS_3)); + + assertTrue(substringVersionConfig.isIncluded(SUBSTRING_INCLUDED_VERSIONS_1)); + assertTrue(substringVersionConfig.isIncluded(SUBSTRING_INCLUDED_VERSIONS_2)); + + } + @Test public void clearCache() throws Exception { sampleRepo.init(); @@ -165,7 +217,7 @@ public void clearCache() throws Exception { new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); config.setDefaultVersion("master"); config.setImplicit(true); - config.setCachingConfiguration(new LibraryCachingConfiguration(30, null)); + config.setCachingConfiguration(new LibraryCachingConfiguration(30, null, "master")); GlobalLibraries.get().getLibraries().add(config); // Run build and check that cache gets created. WorkflowJob p = r.createProject(WorkflowJob.class); @@ -182,4 +234,29 @@ public void clearCache() throws Exception { assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile())); } + @Test + public void clearCacheConflict() throws Exception { + sampleRepo.init(); + sampleRepo.write("vars/foo.groovy", "def call() { echo 'foo' }"); + sampleRepo.git("add", "vars"); + sampleRepo.git("commit", "--message=init"); + LibraryConfiguration config = new LibraryConfiguration("library", + new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); + config.setDefaultVersion("master"); + config.setImplicit(true); + // Same version specified in both include and exclude version + //Exclude takes precedence + config.setCachingConfiguration(new LibraryCachingConfiguration(30, "master", "master")); + GlobalLibraries.get().getLibraries().add(config); + // Run build and check that cache gets created. + WorkflowJob p = r.createProject(WorkflowJob.class); + p.setDefinition(new CpsFlowDefinition("foo()", true)); + WorkflowRun b = r.buildAndAssertSuccess(p); + LibrariesAction action = b.getAction(LibrariesAction.class); + LibraryRecord record = action.getLibraries().get(0); + FilePath cache = LibraryCachingConfiguration.getGlobalLibrariesCacheDir().child(record.getDirectoryName()); + assertThat(new File(cache.getRemote()), not(anExistingDirectory())); + assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile())); + } + } diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java index c1faefca..23d2b431 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java @@ -71,7 +71,7 @@ public class ResourceStepTest { initFixedContentLibrary(); LibraryConfiguration libraryConfig = new LibraryConfiguration("stuff", new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); - libraryConfig.setCachingConfiguration(new LibraryCachingConfiguration(0, "")); + libraryConfig.setCachingConfiguration(new LibraryCachingConfiguration(0, "","")); GlobalLibraries.get().setLibraries(Collections.singletonList(libraryConfig)); WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); @@ -92,7 +92,7 @@ public class ResourceStepTest { initFixedContentLibrary(); LibraryConfiguration libraryConfig = new LibraryConfiguration("stuff", new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); - libraryConfig.setCachingConfiguration(new LibraryCachingConfiguration(0, "test_unused other")); + libraryConfig.setCachingConfiguration(new LibraryCachingConfiguration(0, "test_unused other", null)); GlobalLibraries.get().setLibraries(Collections.singletonList(libraryConfig)); WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); @@ -114,7 +114,7 @@ public class ResourceStepTest { initFixedContentLibrary(); LibraryConfiguration libraryConfig = new LibraryConfiguration("stuff", new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); - libraryConfig.setCachingConfiguration(new LibraryCachingConfiguration(60, "test_unused other")); + libraryConfig.setCachingConfiguration(new LibraryCachingConfiguration(60, "test_unused other", "")); GlobalLibraries.get().setLibraries(Collections.singletonList(libraryConfig)); WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); From f29d6cd203d9583f70354fe0ad78a1a04a20bf2f Mon Sep 17 00:00:00 2001 From: "KINAXIS\\agharat" Date: Tue, 16 Aug 2022 10:22:16 -0400 Subject: [PATCH 2/8] Updated code for deleting specific cache version --- .../plugins/workflow/libs/LibraryAdder.java | 6 +- .../workflow/libs/LibraryCachingCleanup.java | 22 ++-- .../libs/LibraryCachingConfiguration.java | 113 ++++++++---------- .../plugins/workflow/libs/LibraryRecord.java | 3 +- .../LibraryCachingConfiguration/config.jelly | 9 +- .../help-cachedLibraryRef.html | 3 + .../help-includedVersionsStr.html | 5 +- .../workflow/libs/LibraryAdderTest.java | 2 +- .../libs/LibraryCachingCleanupTest.java | 6 +- .../libs/LibraryCachingConfigurationTest.java | 91 +++++++++++--- .../workflow/libs/LibraryStepTest.java | 7 +- .../workflow/libs/ResourceStepTest.java | 11 +- 12 files changed, 167 insertions(+), 111 deletions(-) create mode 100644 src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-cachedLibraryRef.html diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryAdder.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryAdder.java index 1b7b879a..68675de8 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryAdder.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryAdder.java @@ -202,7 +202,7 @@ static List retrieve(@NonNull LibraryRecord record, @NonNull LibraryRetriev FilePath libDir = new FilePath(execution.getOwner().getRootDir()).child("libs/" + record.getDirectoryName()); Boolean shouldCache = cachingConfiguration != null; final FilePath versionCacheDir = new FilePath(LibraryCachingConfiguration.getGlobalLibrariesCacheDir(), record.getDirectoryName()); - ReentrantReadWriteLock retrieveLock = getReadWriteLockFor(record.getDirectoryName()); + ReentrantReadWriteLock retrieveLock = getReadWriteLockFor(record.getName()); final FilePath lastReadFile = new FilePath(versionCacheDir, LibraryCachingConfiguration.LAST_READ_FILE); if(shouldCache && cachingConfiguration.isExcluded(version)) { @@ -238,7 +238,7 @@ static List retrieve(@NonNull LibraryRecord record, @NonNull LibraryRetriev } if (retrieve) { - listener.getLogger().println("Caching library " + name + "@" + version); + listener.getLogger().println("Caching library " + name + "@" + version); versionCacheDir.mkdirs(); retriever.retrieve(name, version, changelog, versionCacheDir, run, listener); } @@ -251,7 +251,7 @@ static List retrieve(@NonNull LibraryRecord record, @NonNull LibraryRetriev } lastReadFile.touch(System.currentTimeMillis()); - versionCacheDir.withSuffix("-name.txt").write(name, "UTF-8"); + versionCacheDir.withSuffix("-name.txt").write(name + "@" + version, "UTF-8"); versionCacheDir.copyRecursiveTo(libDir); } finally { retrieveLock.readLock().unlock(); diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanup.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanup.java index 591dcc0a..ea14cb5f 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanup.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanup.java @@ -28,14 +28,13 @@ public LibraryCachingCleanup() { @Override protected void execute(TaskListener listener) throws IOException, InterruptedException { FilePath globalCacheDir = LibraryCachingConfiguration.getGlobalLibrariesCacheDir(); for (FilePath library : globalCacheDir.list()) { - if (!removeIfExpiredCacheDirectory(library)) { - // Prior to the SECURITY-2586 fix, library caches had a two-level directory structure. - // These caches will never be used again, so we delete any that we find. - for (FilePath version: library.list()) { - if (version.child(LibraryCachingConfiguration.LAST_READ_FILE).exists()) { - library.deleteRecursive(); - break; + for (FilePath versionDir : library.listDirectories()) { + if (!removeIfExpiredCacheDirectory(versionDir)) { + FilePath parent = versionDir.getParent(); + if (parent != null) { + parent.deleteRecursive(); } + break; } } } @@ -48,14 +47,15 @@ public LibraryCachingCleanup() { */ private boolean removeIfExpiredCacheDirectory(FilePath library) throws IOException, InterruptedException { final FilePath lastReadFile = new FilePath(library, LibraryCachingConfiguration.LAST_READ_FILE); - if (lastReadFile.exists()) { + if (lastReadFile.exists() && library.withSuffix("-name.txt").exists()) { ReentrantReadWriteLock retrieveLock = LibraryAdder.getReadWriteLockFor(library.getName()); retrieveLock.writeLock().lockInterruptibly(); try { if (System.currentTimeMillis() - lastReadFile.lastModified() > TimeUnit.DAYS.toMillis(EXPIRE_AFTER_READ_DAYS)) { - - library.deleteRecursive(); - library.withSuffix("-name.txt").delete(); + FilePath parent = library.getParent(); + if (parent != null) { + parent.deleteRecursive(); + } } } finally { retrieveLock.writeLock().unlock(); diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java index 944033a7..83f52a58 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java @@ -31,6 +31,7 @@ public final class LibraryCachingConfiguration extends AbstractDescribableImpl getExcludedVersions() { if (excludedVersionsStr == null) { @@ -117,78 +121,61 @@ public static FilePath getGlobalLibrariesCacheDir() { } @Extension public static class DescriptorImpl extends Descriptor { - public FormValidation doClearCache(@QueryParameter String name, @QueryParameter boolean forceDelete) throws InterruptedException { + public FormValidation doClearCache(@QueryParameter String name, @QueryParameter String cachedLibraryRef, @QueryParameter boolean forceDelete) throws InterruptedException { Jenkins.get().checkPermission(Jenkins.ADMINISTER); - + String cacheDirName = null; try { if (LibraryCachingConfiguration.getGlobalLibrariesCacheDir().exists()) { - for (FilePath libraryNamePath : LibraryCachingConfiguration.getGlobalLibrariesCacheDir().list("*-name.txt")) { - // Libraries configured in distinct locations may have the same name. Since only admins are allowed here, this is not a huge issue, but it is probably unexpected. - String cacheName; - try (InputStream stream = libraryNamePath.read()) { - cacheName = IOUtils.toString(stream, StandardCharsets.UTF_8); - } - if (cacheName.equals(name)) { - FilePath libraryCachePath = LibraryCachingConfiguration.getGlobalLibrariesCacheDir() - .child(libraryNamePath.getName().replace("-name.txt", "")); - if (forceDelete) { - LOGGER.log(Level.FINER, "Force deleting cache for {0}", name); - libraryCachePath.deleteRecursive(); - libraryNamePath.delete(); - } else { - LOGGER.log(Level.FINER, "Safe deleting cache for {0}", name); - ReentrantReadWriteLock retrieveLock = LibraryAdder.getReadWriteLockFor(libraryCachePath.getName()); - if (retrieveLock.writeLock().tryLock(10, TimeUnit.SECONDS)) { - try { - libraryCachePath.deleteRecursive(); - libraryNamePath.delete(); - } finally { - retrieveLock.writeLock().unlock(); + outer: for (FilePath libraryCache : LibraryCachingConfiguration.getGlobalLibrariesCacheDir().listDirectories()) { + for (FilePath libraryNamePath : libraryCache.list("*-name.txt")) { + if (libraryNamePath.readToString().startsWith(name + "@")) { + FilePath libraryCachePath = libraryNamePath.getParent(); + if (libraryCachePath != null) { + FilePath versionCachePath = new FilePath(libraryCachePath, libraryNamePath.getName().replace("-name.txt", "")); + LOGGER.log(Level.FINER, "Safe deleting cache for {0}", name); + ReentrantReadWriteLock retrieveLock = LibraryAdder.getReadWriteLockFor(libraryCachePath.getName()); + if (forceDelete || retrieveLock.writeLock().tryLock(10, TimeUnit.SECONDS)) { + if (forceDelete) { + LOGGER.log(Level.FINER, "Force deleting cache for {0}", name); + } else { + LOGGER.log(Level.FINER, "Safe deleting cache for {0}", name); + } + try { + if (StringUtils.isNotEmpty(cachedLibraryRef)) { + if (libraryNamePath.readToString().equals(name + "@" + cachedLibraryRef)) { + cacheDirName = name + "@" + cachedLibraryRef; + libraryNamePath.delete(); + versionCachePath.deleteRecursive(); + break outer; + } + } else { + cacheDirName = name; + libraryCachePath.deleteRecursive(); + break outer; + } + } finally { + if (!forceDelete) { + retrieveLock.writeLock().unlock(); + } + } + } else { + return FormValidation.error("The cache dir could not be deleted because it is currently being used by another thread. Please try again."); } - } else { - return FormValidation.error("The cache dir could not be deleted because it is currently being used by another thread. Please try again."); } } } } } } catch (IOException ex) { - return FormValidation.error(ex, "The cache dir was not deleted successfully"); + return FormValidation.error(ex, String.format("The cache dir %s was not deleted successfully", cacheDirName)); } - return FormValidation.ok("The cache dir was deleted successfully."); - } - } - - /** - * Method can be called from an external source to delete cache of a particular version of the shared library - * Example: delete cache from through pipeline script once the build is successful - * @param jenkinsLibrary Name of the shared library - * @param version Name of the version to delete the cache - * @throws IOException - * @throws InterruptedException - */ - public static void deleteLibraryCacheForBranch(String jenkinsLibrary, String version) throws IOException, InterruptedException { - if (LibraryCachingConfiguration.getGlobalLibrariesCacheDir().exists()) { - for (FilePath libraryNamePath : LibraryCachingConfiguration.getGlobalLibrariesCacheDir().list("*-name.txt")) { - String cacheName; - try (InputStream stream = libraryNamePath.read()) { - cacheName = IOUtils.toString(stream, StandardCharsets.UTF_8); - - if (cacheName.equals(jenkinsLibrary)) { - FilePath libraryCachePath = LibraryCachingConfiguration.getGlobalLibrariesCacheDir() - .child(libraryNamePath.getName().replace("-name.txt", "")); - ReentrantReadWriteLock retrieveLock = LibraryAdder.getReadWriteLockFor(libraryCachePath.getName()); - if (retrieveLock.writeLock().tryLock(10, TimeUnit.SECONDS)) { - try { - libraryCachePath.deleteRecursive(); - libraryNamePath.delete(); - } finally { - retrieveLock.writeLock().unlock(); - } - } - } - } + + if (cacheDirName == null) { + return FormValidation.ok(String.format("The version %s was not found for library %s.", cachedLibraryRef, name)); + } else { + return FormValidation.ok(String.format("The cache dir %s was deleted successfully.", cacheDirName)); } } + } -} \ No newline at end of file +} diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryRecord.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryRecord.java index c26de4cb..82f34f2e 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryRecord.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryRecord.java @@ -24,6 +24,7 @@ package org.jenkinsci.plugins.workflow.libs; +import java.io.File; import java.util.Collections; import java.util.Set; import java.util.TreeSet; @@ -62,7 +63,7 @@ public final class LibraryRecord { this.trusted = trusted; this.changelog = changelog; this.cachingConfiguration = cachingConfiguration; - this.directoryName = directoryNameFor(name, version, String.valueOf(trusted), source); + this.directoryName = directoryNameFor(name, String.valueOf(trusted), source) + File.separator + directoryNameFor(version); } @Exported diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/config.jelly b/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/config.jelly index 8219f0c7..018ab859 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/config.jelly +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/config.jelly @@ -32,12 +32,15 @@ THE SOFTWARE. - + - + + + + - + diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-cachedLibraryRef.html b/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-cachedLibraryRef.html new file mode 100644 index 00000000..eaa6889d --- /dev/null +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-cachedLibraryRef.html @@ -0,0 +1,3 @@ +
+ Specifies a specific version to clear the cache for. An empty value will clear the cache for all versions. +
\ No newline at end of file diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-includedVersionsStr.html b/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-includedVersionsStr.html index 27365936..1c81e31f 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-includedVersionsStr.html +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-includedVersionsStr.html @@ -1,6 +1,3 @@
- Space separated list of versions to include to allow caching via substring search using .contains() method. Ex: "release/ master". -
-
- Note: Excluded versions will always take precedence over included versions + Space separated list of versions to include to allow caching via substring search using .contains() method. Ex: "release/ master".
\ No newline at end of file diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryAdderTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryAdderTest.java index 3ef953b8..b878e429 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryAdderTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryAdderTest.java @@ -493,7 +493,7 @@ public void parallelBuildsDontInterfereWithExpiredCache() throws Throwable { new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); config.setDefaultVersion("master"); config.setImplicit(true); - config.setCachingConfiguration(new LibraryCachingConfiguration(30, null,null)); + config.setCachingConfiguration(new LibraryCachingConfiguration(30, null, null)); GlobalLibraries.get().getLibraries().add(config); WorkflowJob p1 = r.createProject(WorkflowJob.class); WorkflowJob p2 = r.createProject(WorkflowJob.class); diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanupTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanupTest.java index 9a66cb71..473a129e 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanupTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanupTest.java @@ -71,14 +71,16 @@ public void smokes() throws Throwable { assertThat(new File(cache.getRemote()), anExistingDirectory()); // Run LibraryCachingCleanup and show that cache is not deleted. ExtensionList.lookupSingleton(LibraryCachingCleanup.class).execute(StreamTaskListener.fromStderr()); + assertThat(new File(cache.getParent().getRemote()), anExistingDirectory()); assertThat(new File(cache.getRemote()), anExistingDirectory()); assertThat(new File(cache.withSuffix("-name.txt").getRemote()), anExistingFile()); // Run LibraryCachingCleanup after modifying LAST_READ_FILE to be an old date and and show that cache is deleted. long oldMillis = ZonedDateTime.now().minusDays(LibraryCachingCleanup.EXPIRE_AFTER_READ_DAYS + 1).toInstant().toEpochMilli(); cache.child(LibraryCachingConfiguration.LAST_READ_FILE).touch(oldMillis); ExtensionList.lookupSingleton(LibraryCachingCleanup.class).execute(StreamTaskListener.fromStderr()); - assertThat(new File(cache.getRemote()), not(anExistingDirectory())); - assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingDirectory())); + assertThat(new File(cache.getParent().getRemote()), not(anExistingDirectory())); + assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile())); + } @Test diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java index 95fe8d42..880f9092 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java @@ -27,6 +27,8 @@ import hudson.ExtensionList; import hudson.FilePath; import java.io.File; +import java.util.Arrays; +import java.util.List; import jenkins.plugins.git.GitSCMSource; import jenkins.plugins.git.GitSampleRepoRule; import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition; @@ -61,19 +63,16 @@ public class LibraryCachingConfigurationTest { private static int NO_REFRESH_TIME_MINUTES = 0; private static String NULL_EXCLUDED_VERSION = null; + private static String NULL_INCLUDED_VERSION = null; private static String ONE_EXCLUDED_VERSION = "branch-1"; - - private static String ONE_INCLUDED_VERSION = "branch-1i"; + private static String ONE_INCLUDED_VERSION = "branch-1"; private static String MULTIPLE_EXCLUDED_VERSIONS_1 = "main"; - private static String MULTIPLE_INCLUDED_VERSIONS_1 = "master"; - private static String MULTIPLE_EXCLUDED_VERSIONS_2 = "branch-2"; - private static String MULTIPLE_INCLUDED_VERSIONS_2 = "branch-2i"; private static String MULTIPLE_EXCLUDED_VERSIONS_3 = "branch-3"; @@ -81,7 +80,6 @@ public class LibraryCachingConfigurationTest { private static String SUBSTRING_EXCLUDED_VERSIONS_1 = "feature/test-substring-exclude"; - private static String SUBSTRING_INCLUDED_VERSIONS_1 = "feature_include/test-substring"; private static String SUBSTRING_EXCLUDED_VERSIONS_2 = "test-other-substring-exclude"; @@ -105,6 +103,7 @@ public class LibraryCachingConfigurationTest { "feature_include/ other-substring"; private static String NEVER_EXCLUDED_VERSION = "never-excluded-version"; + private static String NEVER_INCLUDED_VERSION = "never-included-version"; @Before public void createCachingConfiguration() { @@ -142,6 +141,7 @@ public void isRefreshEnabled() { assertFalse(oneVersionConfig.isRefreshEnabled()); } + @Test @WithoutJenkins public void getExcludedVersionsStr() { @@ -151,6 +151,8 @@ public void getExcludedVersionsStr() { assertThat(substringVersionConfig.getExcludedVersionsStr(), is(SUBSTRING_EXCLUDED_VERSIONS)); } + + @Issue("JENKINS-69135") //"Versions to include" feature for caching @Test @WithoutJenkins public void getIncludedVersionsStr() { @@ -160,6 +162,7 @@ public void getIncludedVersionsStr() { assertThat(substringVersionConfig.getIncludedVersionsStr(), is(SUBSTRING_INCLUDED_VERSIONS)); } + @Test @WithoutJenkins public void isExcluded() { @@ -190,6 +193,7 @@ public void isExcluded() { assertFalse(substringVersionConfig.isExcluded(null)); } + @Issue("JENKINS-69135") //"Versions to include" feature for caching @Test @WithoutJenkins public void isIncluded() { @@ -205,35 +209,90 @@ public void isIncluded() { assertTrue(substringVersionConfig.isIncluded(SUBSTRING_INCLUDED_VERSIONS_1)); assertTrue(substringVersionConfig.isIncluded(SUBSTRING_INCLUDED_VERSIONS_2)); + assertFalse(nullVersionConfig.isIncluded(NEVER_EXCLUDED_VERSION)); + assertFalse(oneVersionConfig.isIncluded(NEVER_EXCLUDED_VERSION)); + assertFalse(multiVersionConfig.isIncluded(NEVER_EXCLUDED_VERSION)); + + assertFalse(nullVersionConfig.isIncluded("")); + assertFalse(oneVersionConfig.isIncluded("")); + assertFalse(multiVersionConfig.isIncluded("")); + assertFalse(substringVersionConfig.isIncluded("")); + + assertFalse(nullVersionConfig.isIncluded(null)); + assertFalse(oneVersionConfig.isIncluded(null)); + assertFalse(multiVersionConfig.isIncluded(null)); + assertFalse(substringVersionConfig.isIncluded(null)); } @Test public void clearCache() throws Exception { + List caches = setupLibraryCaches(); + FilePath cache = caches.get(0); + FilePath cache2 = caches.get(1); + assertThat("Must be different paths", cache, not(equalTo(cache2))); + assertThat(new File(cache.getParent().getRemote()), anExistingDirectory()); + assertThat(new File(cache.getRemote()), anExistingDirectory()); + assertThat(new File(cache2.getRemote()), anExistingDirectory()); + assertThat(new File(cache.withSuffix("-name.txt").getRemote()), anExistingFile()); + assertThat(cache.withSuffix("-name.txt").readToString(), equalTo("library@master")); + assertThat(cache2.withSuffix("-name.txt").readToString(), equalTo("library@feature/something")); + // Clear the cache. TODO: Would be more realistic to set up security and use WebClient. + ExtensionList.lookupSingleton(LibraryCachingConfiguration.DescriptorImpl.class).doClearCache("library", "", false); + assertThat(new File(cache.getParent().getRemote()), not(anExistingDirectory())); + assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile())); + } + + @Test + public void clearCacheVersion() throws Exception { + + List caches = setupLibraryCaches(); + FilePath cache = caches.get(0); + FilePath cache2 = caches.get(1); + assertThat(new File(cache.getRemote()), anExistingDirectory()); + // Clear the cache. TODO: Would be more realistic to set up security and use WebClient. + ExtensionList.lookupSingleton(LibraryCachingConfiguration.DescriptorImpl.class).doClearCache("library", "master", false); + assertThat(new File(cache.getParent().getRemote()), anExistingDirectory()); + assertThat(new File(cache.getRemote()), not(anExistingDirectory())); + assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile())); + //Other cache has not been touched + assertThat(new File(cache2.getRemote()), anExistingDirectory()); + assertThat(new File(cache2.withSuffix("-name.txt").getRemote()), anExistingFile()); + } + + + private List setupLibraryCaches() throws Exception { sampleRepo.init(); sampleRepo.write("vars/foo.groovy", "def call() { echo 'foo' }"); sampleRepo.git("add", "vars"); sampleRepo.git("commit", "--message=init"); + sampleRepo.git("branch", "feature/something"); LibraryConfiguration config = new LibraryConfiguration("library", new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); config.setDefaultVersion("master"); - config.setImplicit(true); - config.setCachingConfiguration(new LibraryCachingConfiguration(30, null, "master")); + config.setImplicit(false); + config.setCachingConfiguration(new LibraryCachingConfiguration(30, null, "master feature/something")); + config.setAllowVersionOverride(true); GlobalLibraries.get().getLibraries().add(config); // Run build and check that cache gets created. WorkflowJob p = r.createProject(WorkflowJob.class); - p.setDefinition(new CpsFlowDefinition("foo()", true)); + p.setDefinition(new CpsFlowDefinition("library identifier: 'library', changelog:false\n\nfoo()", true)); WorkflowRun b = r.buildAndAssertSuccess(p); + WorkflowJob p2 = r.createProject(WorkflowJob.class); + p2.setDefinition(new CpsFlowDefinition("library identifier: 'library@feature/something', changelog:false\n\nfoo()", true)); + WorkflowRun b2 = r.buildAndAssertSuccess(p2); LibrariesAction action = b.getAction(LibrariesAction.class); LibraryRecord record = action.getLibraries().get(0); + LibrariesAction action2 = b2.getAction(LibrariesAction.class); + LibraryRecord record2 = action2.getLibraries().get(0); + FilePath cache = LibraryCachingConfiguration.getGlobalLibrariesCacheDir().child(record.getDirectoryName()); - assertThat(new File(cache.getRemote()), anExistingDirectory()); - assertThat(new File(cache.withSuffix("-name.txt").getRemote()), anExistingFile()); - // Clear the cache. TODO: Would be more realistic to set up security and use WebClient. - ExtensionList.lookupSingleton(LibraryCachingConfiguration.DescriptorImpl.class).doClearCache("library", false); - assertThat(new File(cache.getRemote()), not(anExistingDirectory())); - assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile())); + FilePath cache2 = LibraryCachingConfiguration.getGlobalLibrariesCacheDir().child(record2.getDirectoryName()); + + return Arrays.asList(cache, cache2); } + //Test similar substrings in "Versions to include" & "Versions to exclude" + //Exclusion takes precedence @Test public void clearCacheConflict() throws Exception { sampleRepo.init(); @@ -255,8 +314,8 @@ public void clearCacheConflict() throws Exception { LibrariesAction action = b.getAction(LibrariesAction.class); LibraryRecord record = action.getLibraries().get(0); FilePath cache = LibraryCachingConfiguration.getGlobalLibrariesCacheDir().child(record.getDirectoryName()); + // Cache should not get created since the version is included in "Versions to exclude" assertThat(new File(cache.getRemote()), not(anExistingDirectory())); assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile())); } - } diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryStepTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryStepTest.java index 163c8b14..7f3b68fb 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryStepTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryStepTest.java @@ -33,6 +33,8 @@ import hudson.plugins.git.SubmoduleConfig; import hudson.plugins.git.UserRemoteConfig; import hudson.plugins.git.extensions.GitSCMExtension; + +import java.io.File; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -99,14 +101,15 @@ public class LibraryStepTest { r.assertLogContains("ran library", b); LibrariesAction action = b.getAction(LibrariesAction.class); assertNotNull(action); - String directoryName = LibraryRecord.directoryNameFor("stuff", "master", String.valueOf(true), GlobalLibraries.ForJob.class.getName()); + String directoryName = LibraryRecord.directoryNameFor("stuff", String.valueOf(true), GlobalLibraries.ForJob.class.getName()) + File.separator + LibraryRecord.directoryNameFor("master"); + assertEquals("[LibraryRecord{name=stuff, version=master, variables=[x], trusted=true, changelog=true, cachingConfiguration=null, directoryName=" + directoryName + "}]", action.getLibraries().toString()); p.setDefinition(new CpsFlowDefinition("library identifier: 'otherstuff@master', retriever: modernSCM([$class: 'GitSCMSource', remote: $/" + sampleRepo + "/$, credentialsId: '']), changelog: false; x()", true)); b = r.buildAndAssertSuccess(p); r.assertLogContains("ran library", b); action = b.getAction(LibrariesAction.class); assertNotNull(action); - directoryName = LibraryRecord.directoryNameFor("otherstuff", "master", String.valueOf(false), LibraryStep.class.getName() + " " + b.getExternalizableId()); + directoryName = LibraryRecord.directoryNameFor("otherstuff", String.valueOf(false), LibraryStep.class.getName() + " " + b.getExternalizableId()) + File.separator + LibraryRecord.directoryNameFor("master"); assertEquals("[LibraryRecord{name=otherstuff, version=master, variables=[x], trusted=false, changelog=false, cachingConfiguration=null, directoryName=" + directoryName + "}]", action.getLibraries().toString()); } diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java index 23d2b431..27e47207 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java @@ -71,7 +71,7 @@ public class ResourceStepTest { initFixedContentLibrary(); LibraryConfiguration libraryConfig = new LibraryConfiguration("stuff", new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); - libraryConfig.setCachingConfiguration(new LibraryCachingConfiguration(0, "","")); + libraryConfig.setCachingConfiguration(new LibraryCachingConfiguration(0, "", "")); GlobalLibraries.get().setLibraries(Collections.singletonList(libraryConfig)); WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); @@ -92,7 +92,7 @@ public class ResourceStepTest { initFixedContentLibrary(); LibraryConfiguration libraryConfig = new LibraryConfiguration("stuff", new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); - libraryConfig.setCachingConfiguration(new LibraryCachingConfiguration(0, "test_unused other", null)); + libraryConfig.setCachingConfiguration(new LibraryCachingConfiguration(0, "test_unused other", "")); GlobalLibraries.get().setLibraries(Collections.singletonList(libraryConfig)); WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); @@ -195,7 +195,7 @@ public class ResourceStepTest { Path resourcesDir = Paths.get(sampleRepo.getRoot().getPath(), "resources"); Files.createDirectories(resourcesDir); Path symlinkPath = Paths.get(resourcesDir.toString(), "master.key"); - Files.createSymbolicLink(symlinkPath, Paths.get("../../../../../../../secrets/master.key")); + Files.createSymbolicLink(symlinkPath, Paths.get("../../../../../../../../secrets/master.key")); sampleRepo.git("add", "src", "resources"); sampleRepo.git("commit", "--message=init"); @@ -270,8 +270,9 @@ public void clearCache(String name) throws Exception { } public void modifyCacheTimestamp(String name, String version, long timestamp) throws Exception { - String cacheDirName = LibraryRecord.directoryNameFor(name, version, String.valueOf(true), GlobalLibraries.ForJob.class.getName()); - FilePath cacheDir = new FilePath(LibraryCachingConfiguration.getGlobalLibrariesCacheDir(), cacheDirName); + String cacheDirName = LibraryRecord.directoryNameFor(name, String.valueOf(true), GlobalLibraries.ForJob.class.getName()); + FilePath libraryDir = new FilePath(LibraryCachingConfiguration.getGlobalLibrariesCacheDir(), cacheDirName); + FilePath cacheDir = new FilePath(libraryDir, LibraryRecord.directoryNameFor(version)); if (cacheDir.exists()) { cacheDir.touch(timestamp); } From 006be8f043071590668f4cf2e460037248756de2 Mon Sep 17 00:00:00 2001 From: Anisha Gharat <36487349+AnishaGharat@users.noreply.github.com> Date: Tue, 16 Aug 2022 11:59:41 -0400 Subject: [PATCH 3/8] Revert "Updated code for deleting specific cache version" --- .../plugins/workflow/libs/LibraryAdder.java | 6 +- .../workflow/libs/LibraryCachingCleanup.java | 22 ++-- .../libs/LibraryCachingConfiguration.java | 113 ++++++++++-------- .../plugins/workflow/libs/LibraryRecord.java | 3 +- .../LibraryCachingConfiguration/config.jelly | 9 +- .../help-cachedLibraryRef.html | 3 - .../help-includedVersionsStr.html | 5 +- .../workflow/libs/LibraryAdderTest.java | 2 +- .../libs/LibraryCachingCleanupTest.java | 6 +- .../libs/LibraryCachingConfigurationTest.java | 91 +++----------- .../workflow/libs/LibraryStepTest.java | 7 +- .../workflow/libs/ResourceStepTest.java | 11 +- 12 files changed, 111 insertions(+), 167 deletions(-) delete mode 100644 src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-cachedLibraryRef.html diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryAdder.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryAdder.java index 68675de8..1b7b879a 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryAdder.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryAdder.java @@ -202,7 +202,7 @@ static List retrieve(@NonNull LibraryRecord record, @NonNull LibraryRetriev FilePath libDir = new FilePath(execution.getOwner().getRootDir()).child("libs/" + record.getDirectoryName()); Boolean shouldCache = cachingConfiguration != null; final FilePath versionCacheDir = new FilePath(LibraryCachingConfiguration.getGlobalLibrariesCacheDir(), record.getDirectoryName()); - ReentrantReadWriteLock retrieveLock = getReadWriteLockFor(record.getName()); + ReentrantReadWriteLock retrieveLock = getReadWriteLockFor(record.getDirectoryName()); final FilePath lastReadFile = new FilePath(versionCacheDir, LibraryCachingConfiguration.LAST_READ_FILE); if(shouldCache && cachingConfiguration.isExcluded(version)) { @@ -238,7 +238,7 @@ static List retrieve(@NonNull LibraryRecord record, @NonNull LibraryRetriev } if (retrieve) { - listener.getLogger().println("Caching library " + name + "@" + version); + listener.getLogger().println("Caching library " + name + "@" + version); versionCacheDir.mkdirs(); retriever.retrieve(name, version, changelog, versionCacheDir, run, listener); } @@ -251,7 +251,7 @@ static List retrieve(@NonNull LibraryRecord record, @NonNull LibraryRetriev } lastReadFile.touch(System.currentTimeMillis()); - versionCacheDir.withSuffix("-name.txt").write(name + "@" + version, "UTF-8"); + versionCacheDir.withSuffix("-name.txt").write(name, "UTF-8"); versionCacheDir.copyRecursiveTo(libDir); } finally { retrieveLock.readLock().unlock(); diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanup.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanup.java index ea14cb5f..591dcc0a 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanup.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanup.java @@ -28,13 +28,14 @@ public LibraryCachingCleanup() { @Override protected void execute(TaskListener listener) throws IOException, InterruptedException { FilePath globalCacheDir = LibraryCachingConfiguration.getGlobalLibrariesCacheDir(); for (FilePath library : globalCacheDir.list()) { - for (FilePath versionDir : library.listDirectories()) { - if (!removeIfExpiredCacheDirectory(versionDir)) { - FilePath parent = versionDir.getParent(); - if (parent != null) { - parent.deleteRecursive(); + if (!removeIfExpiredCacheDirectory(library)) { + // Prior to the SECURITY-2586 fix, library caches had a two-level directory structure. + // These caches will never be used again, so we delete any that we find. + for (FilePath version: library.list()) { + if (version.child(LibraryCachingConfiguration.LAST_READ_FILE).exists()) { + library.deleteRecursive(); + break; } - break; } } } @@ -47,15 +48,14 @@ public LibraryCachingCleanup() { */ private boolean removeIfExpiredCacheDirectory(FilePath library) throws IOException, InterruptedException { final FilePath lastReadFile = new FilePath(library, LibraryCachingConfiguration.LAST_READ_FILE); - if (lastReadFile.exists() && library.withSuffix("-name.txt").exists()) { + if (lastReadFile.exists()) { ReentrantReadWriteLock retrieveLock = LibraryAdder.getReadWriteLockFor(library.getName()); retrieveLock.writeLock().lockInterruptibly(); try { if (System.currentTimeMillis() - lastReadFile.lastModified() > TimeUnit.DAYS.toMillis(EXPIRE_AFTER_READ_DAYS)) { - FilePath parent = library.getParent(); - if (parent != null) { - parent.deleteRecursive(); - } + + library.deleteRecursive(); + library.withSuffix("-name.txt").delete(); } } finally { retrieveLock.writeLock().unlock(); diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java index 83f52a58..944033a7 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java @@ -31,7 +31,6 @@ public final class LibraryCachingConfiguration extends AbstractDescribableImpl getExcludedVersions() { if (excludedVersionsStr == null) { @@ -121,61 +117,78 @@ public static FilePath getGlobalLibrariesCacheDir() { } @Extension public static class DescriptorImpl extends Descriptor { - public FormValidation doClearCache(@QueryParameter String name, @QueryParameter String cachedLibraryRef, @QueryParameter boolean forceDelete) throws InterruptedException { + public FormValidation doClearCache(@QueryParameter String name, @QueryParameter boolean forceDelete) throws InterruptedException { Jenkins.get().checkPermission(Jenkins.ADMINISTER); - String cacheDirName = null; + try { if (LibraryCachingConfiguration.getGlobalLibrariesCacheDir().exists()) { - outer: for (FilePath libraryCache : LibraryCachingConfiguration.getGlobalLibrariesCacheDir().listDirectories()) { - for (FilePath libraryNamePath : libraryCache.list("*-name.txt")) { - if (libraryNamePath.readToString().startsWith(name + "@")) { - FilePath libraryCachePath = libraryNamePath.getParent(); - if (libraryCachePath != null) { - FilePath versionCachePath = new FilePath(libraryCachePath, libraryNamePath.getName().replace("-name.txt", "")); - LOGGER.log(Level.FINER, "Safe deleting cache for {0}", name); - ReentrantReadWriteLock retrieveLock = LibraryAdder.getReadWriteLockFor(libraryCachePath.getName()); - if (forceDelete || retrieveLock.writeLock().tryLock(10, TimeUnit.SECONDS)) { - if (forceDelete) { - LOGGER.log(Level.FINER, "Force deleting cache for {0}", name); - } else { - LOGGER.log(Level.FINER, "Safe deleting cache for {0}", name); - } - try { - if (StringUtils.isNotEmpty(cachedLibraryRef)) { - if (libraryNamePath.readToString().equals(name + "@" + cachedLibraryRef)) { - cacheDirName = name + "@" + cachedLibraryRef; - libraryNamePath.delete(); - versionCachePath.deleteRecursive(); - break outer; - } - } else { - cacheDirName = name; - libraryCachePath.deleteRecursive(); - break outer; - } - } finally { - if (!forceDelete) { - retrieveLock.writeLock().unlock(); - } - } - } else { - return FormValidation.error("The cache dir could not be deleted because it is currently being used by another thread. Please try again."); + for (FilePath libraryNamePath : LibraryCachingConfiguration.getGlobalLibrariesCacheDir().list("*-name.txt")) { + // Libraries configured in distinct locations may have the same name. Since only admins are allowed here, this is not a huge issue, but it is probably unexpected. + String cacheName; + try (InputStream stream = libraryNamePath.read()) { + cacheName = IOUtils.toString(stream, StandardCharsets.UTF_8); + } + if (cacheName.equals(name)) { + FilePath libraryCachePath = LibraryCachingConfiguration.getGlobalLibrariesCacheDir() + .child(libraryNamePath.getName().replace("-name.txt", "")); + if (forceDelete) { + LOGGER.log(Level.FINER, "Force deleting cache for {0}", name); + libraryCachePath.deleteRecursive(); + libraryNamePath.delete(); + } else { + LOGGER.log(Level.FINER, "Safe deleting cache for {0}", name); + ReentrantReadWriteLock retrieveLock = LibraryAdder.getReadWriteLockFor(libraryCachePath.getName()); + if (retrieveLock.writeLock().tryLock(10, TimeUnit.SECONDS)) { + try { + libraryCachePath.deleteRecursive(); + libraryNamePath.delete(); + } finally { + retrieveLock.writeLock().unlock(); } + } else { + return FormValidation.error("The cache dir could not be deleted because it is currently being used by another thread. Please try again."); } } } } } } catch (IOException ex) { - return FormValidation.error(ex, String.format("The cache dir %s was not deleted successfully", cacheDirName)); - } - - if (cacheDirName == null) { - return FormValidation.ok(String.format("The version %s was not found for library %s.", cachedLibraryRef, name)); - } else { - return FormValidation.ok(String.format("The cache dir %s was deleted successfully.", cacheDirName)); + return FormValidation.error(ex, "The cache dir was not deleted successfully"); } + return FormValidation.ok("The cache dir was deleted successfully."); } + } + /** + * Method can be called from an external source to delete cache of a particular version of the shared library + * Example: delete cache from through pipeline script once the build is successful + * @param jenkinsLibrary Name of the shared library + * @param version Name of the version to delete the cache + * @throws IOException + * @throws InterruptedException + */ + public static void deleteLibraryCacheForBranch(String jenkinsLibrary, String version) throws IOException, InterruptedException { + if (LibraryCachingConfiguration.getGlobalLibrariesCacheDir().exists()) { + for (FilePath libraryNamePath : LibraryCachingConfiguration.getGlobalLibrariesCacheDir().list("*-name.txt")) { + String cacheName; + try (InputStream stream = libraryNamePath.read()) { + cacheName = IOUtils.toString(stream, StandardCharsets.UTF_8); + + if (cacheName.equals(jenkinsLibrary)) { + FilePath libraryCachePath = LibraryCachingConfiguration.getGlobalLibrariesCacheDir() + .child(libraryNamePath.getName().replace("-name.txt", "")); + ReentrantReadWriteLock retrieveLock = LibraryAdder.getReadWriteLockFor(libraryCachePath.getName()); + if (retrieveLock.writeLock().tryLock(10, TimeUnit.SECONDS)) { + try { + libraryCachePath.deleteRecursive(); + libraryNamePath.delete(); + } finally { + retrieveLock.writeLock().unlock(); + } + } + } + } + } + } } -} +} \ No newline at end of file diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryRecord.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryRecord.java index 82f34f2e..c26de4cb 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryRecord.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryRecord.java @@ -24,7 +24,6 @@ package org.jenkinsci.plugins.workflow.libs; -import java.io.File; import java.util.Collections; import java.util.Set; import java.util.TreeSet; @@ -63,7 +62,7 @@ public final class LibraryRecord { this.trusted = trusted; this.changelog = changelog; this.cachingConfiguration = cachingConfiguration; - this.directoryName = directoryNameFor(name, String.valueOf(trusted), source) + File.separator + directoryNameFor(version); + this.directoryName = directoryNameFor(name, version, String.valueOf(trusted), source); } @Exported diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/config.jelly b/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/config.jelly index 018ab859..8219f0c7 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/config.jelly +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/config.jelly @@ -32,15 +32,12 @@ THE SOFTWARE. - + - - - - + - + diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-cachedLibraryRef.html b/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-cachedLibraryRef.html deleted file mode 100644 index eaa6889d..00000000 --- a/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-cachedLibraryRef.html +++ /dev/null @@ -1,3 +0,0 @@ -
- Specifies a specific version to clear the cache for. An empty value will clear the cache for all versions. -
\ No newline at end of file diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-includedVersionsStr.html b/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-includedVersionsStr.html index 1c81e31f..27365936 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-includedVersionsStr.html +++ b/src/main/resources/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration/help-includedVersionsStr.html @@ -1,3 +1,6 @@
- Space separated list of versions to include to allow caching via substring search using .contains() method. Ex: "release/ master". + Space separated list of versions to include to allow caching via substring search using .contains() method. Ex: "release/ master". +
+
+ Note: Excluded versions will always take precedence over included versions
\ No newline at end of file diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryAdderTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryAdderTest.java index b878e429..3ef953b8 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryAdderTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryAdderTest.java @@ -493,7 +493,7 @@ public void parallelBuildsDontInterfereWithExpiredCache() throws Throwable { new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); config.setDefaultVersion("master"); config.setImplicit(true); - config.setCachingConfiguration(new LibraryCachingConfiguration(30, null, null)); + config.setCachingConfiguration(new LibraryCachingConfiguration(30, null,null)); GlobalLibraries.get().getLibraries().add(config); WorkflowJob p1 = r.createProject(WorkflowJob.class); WorkflowJob p2 = r.createProject(WorkflowJob.class); diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanupTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanupTest.java index 473a129e..9a66cb71 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanupTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanupTest.java @@ -71,16 +71,14 @@ public void smokes() throws Throwable { assertThat(new File(cache.getRemote()), anExistingDirectory()); // Run LibraryCachingCleanup and show that cache is not deleted. ExtensionList.lookupSingleton(LibraryCachingCleanup.class).execute(StreamTaskListener.fromStderr()); - assertThat(new File(cache.getParent().getRemote()), anExistingDirectory()); assertThat(new File(cache.getRemote()), anExistingDirectory()); assertThat(new File(cache.withSuffix("-name.txt").getRemote()), anExistingFile()); // Run LibraryCachingCleanup after modifying LAST_READ_FILE to be an old date and and show that cache is deleted. long oldMillis = ZonedDateTime.now().minusDays(LibraryCachingCleanup.EXPIRE_AFTER_READ_DAYS + 1).toInstant().toEpochMilli(); cache.child(LibraryCachingConfiguration.LAST_READ_FILE).touch(oldMillis); ExtensionList.lookupSingleton(LibraryCachingCleanup.class).execute(StreamTaskListener.fromStderr()); - assertThat(new File(cache.getParent().getRemote()), not(anExistingDirectory())); - assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile())); - + assertThat(new File(cache.getRemote()), not(anExistingDirectory())); + assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingDirectory())); } @Test diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java index 880f9092..95fe8d42 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java @@ -27,8 +27,6 @@ import hudson.ExtensionList; import hudson.FilePath; import java.io.File; -import java.util.Arrays; -import java.util.List; import jenkins.plugins.git.GitSCMSource; import jenkins.plugins.git.GitSampleRepoRule; import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition; @@ -63,16 +61,19 @@ public class LibraryCachingConfigurationTest { private static int NO_REFRESH_TIME_MINUTES = 0; private static String NULL_EXCLUDED_VERSION = null; - private static String NULL_INCLUDED_VERSION = null; private static String ONE_EXCLUDED_VERSION = "branch-1"; - private static String ONE_INCLUDED_VERSION = "branch-1"; + + private static String ONE_INCLUDED_VERSION = "branch-1i"; private static String MULTIPLE_EXCLUDED_VERSIONS_1 = "main"; + private static String MULTIPLE_INCLUDED_VERSIONS_1 = "master"; + private static String MULTIPLE_EXCLUDED_VERSIONS_2 = "branch-2"; + private static String MULTIPLE_INCLUDED_VERSIONS_2 = "branch-2i"; private static String MULTIPLE_EXCLUDED_VERSIONS_3 = "branch-3"; @@ -80,6 +81,7 @@ public class LibraryCachingConfigurationTest { private static String SUBSTRING_EXCLUDED_VERSIONS_1 = "feature/test-substring-exclude"; + private static String SUBSTRING_INCLUDED_VERSIONS_1 = "feature_include/test-substring"; private static String SUBSTRING_EXCLUDED_VERSIONS_2 = "test-other-substring-exclude"; @@ -103,7 +105,6 @@ public class LibraryCachingConfigurationTest { "feature_include/ other-substring"; private static String NEVER_EXCLUDED_VERSION = "never-excluded-version"; - private static String NEVER_INCLUDED_VERSION = "never-included-version"; @Before public void createCachingConfiguration() { @@ -141,7 +142,6 @@ public void isRefreshEnabled() { assertFalse(oneVersionConfig.isRefreshEnabled()); } - @Test @WithoutJenkins public void getExcludedVersionsStr() { @@ -151,8 +151,6 @@ public void getExcludedVersionsStr() { assertThat(substringVersionConfig.getExcludedVersionsStr(), is(SUBSTRING_EXCLUDED_VERSIONS)); } - - @Issue("JENKINS-69135") //"Versions to include" feature for caching @Test @WithoutJenkins public void getIncludedVersionsStr() { @@ -162,7 +160,6 @@ public void getIncludedVersionsStr() { assertThat(substringVersionConfig.getIncludedVersionsStr(), is(SUBSTRING_INCLUDED_VERSIONS)); } - @Test @WithoutJenkins public void isExcluded() { @@ -193,7 +190,6 @@ public void isExcluded() { assertFalse(substringVersionConfig.isExcluded(null)); } - @Issue("JENKINS-69135") //"Versions to include" feature for caching @Test @WithoutJenkins public void isIncluded() { @@ -209,90 +205,35 @@ public void isIncluded() { assertTrue(substringVersionConfig.isIncluded(SUBSTRING_INCLUDED_VERSIONS_1)); assertTrue(substringVersionConfig.isIncluded(SUBSTRING_INCLUDED_VERSIONS_2)); - assertFalse(nullVersionConfig.isIncluded(NEVER_EXCLUDED_VERSION)); - assertFalse(oneVersionConfig.isIncluded(NEVER_EXCLUDED_VERSION)); - assertFalse(multiVersionConfig.isIncluded(NEVER_EXCLUDED_VERSION)); - - assertFalse(nullVersionConfig.isIncluded("")); - assertFalse(oneVersionConfig.isIncluded("")); - assertFalse(multiVersionConfig.isIncluded("")); - assertFalse(substringVersionConfig.isIncluded("")); - - assertFalse(nullVersionConfig.isIncluded(null)); - assertFalse(oneVersionConfig.isIncluded(null)); - assertFalse(multiVersionConfig.isIncluded(null)); - assertFalse(substringVersionConfig.isIncluded(null)); } @Test public void clearCache() throws Exception { - List caches = setupLibraryCaches(); - FilePath cache = caches.get(0); - FilePath cache2 = caches.get(1); - assertThat("Must be different paths", cache, not(equalTo(cache2))); - assertThat(new File(cache.getParent().getRemote()), anExistingDirectory()); - assertThat(new File(cache.getRemote()), anExistingDirectory()); - assertThat(new File(cache2.getRemote()), anExistingDirectory()); - assertThat(new File(cache.withSuffix("-name.txt").getRemote()), anExistingFile()); - assertThat(cache.withSuffix("-name.txt").readToString(), equalTo("library@master")); - assertThat(cache2.withSuffix("-name.txt").readToString(), equalTo("library@feature/something")); - // Clear the cache. TODO: Would be more realistic to set up security and use WebClient. - ExtensionList.lookupSingleton(LibraryCachingConfiguration.DescriptorImpl.class).doClearCache("library", "", false); - assertThat(new File(cache.getParent().getRemote()), not(anExistingDirectory())); - assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile())); - } - - @Test - public void clearCacheVersion() throws Exception { - - List caches = setupLibraryCaches(); - FilePath cache = caches.get(0); - FilePath cache2 = caches.get(1); - assertThat(new File(cache.getRemote()), anExistingDirectory()); - // Clear the cache. TODO: Would be more realistic to set up security and use WebClient. - ExtensionList.lookupSingleton(LibraryCachingConfiguration.DescriptorImpl.class).doClearCache("library", "master", false); - assertThat(new File(cache.getParent().getRemote()), anExistingDirectory()); - assertThat(new File(cache.getRemote()), not(anExistingDirectory())); - assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile())); - //Other cache has not been touched - assertThat(new File(cache2.getRemote()), anExistingDirectory()); - assertThat(new File(cache2.withSuffix("-name.txt").getRemote()), anExistingFile()); - } - - - private List setupLibraryCaches() throws Exception { sampleRepo.init(); sampleRepo.write("vars/foo.groovy", "def call() { echo 'foo' }"); sampleRepo.git("add", "vars"); sampleRepo.git("commit", "--message=init"); - sampleRepo.git("branch", "feature/something"); LibraryConfiguration config = new LibraryConfiguration("library", new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); config.setDefaultVersion("master"); - config.setImplicit(false); - config.setCachingConfiguration(new LibraryCachingConfiguration(30, null, "master feature/something")); - config.setAllowVersionOverride(true); + config.setImplicit(true); + config.setCachingConfiguration(new LibraryCachingConfiguration(30, null, "master")); GlobalLibraries.get().getLibraries().add(config); // Run build and check that cache gets created. WorkflowJob p = r.createProject(WorkflowJob.class); - p.setDefinition(new CpsFlowDefinition("library identifier: 'library', changelog:false\n\nfoo()", true)); + p.setDefinition(new CpsFlowDefinition("foo()", true)); WorkflowRun b = r.buildAndAssertSuccess(p); - WorkflowJob p2 = r.createProject(WorkflowJob.class); - p2.setDefinition(new CpsFlowDefinition("library identifier: 'library@feature/something', changelog:false\n\nfoo()", true)); - WorkflowRun b2 = r.buildAndAssertSuccess(p2); LibrariesAction action = b.getAction(LibrariesAction.class); LibraryRecord record = action.getLibraries().get(0); - LibrariesAction action2 = b2.getAction(LibrariesAction.class); - LibraryRecord record2 = action2.getLibraries().get(0); - FilePath cache = LibraryCachingConfiguration.getGlobalLibrariesCacheDir().child(record.getDirectoryName()); - FilePath cache2 = LibraryCachingConfiguration.getGlobalLibrariesCacheDir().child(record2.getDirectoryName()); - - return Arrays.asList(cache, cache2); + assertThat(new File(cache.getRemote()), anExistingDirectory()); + assertThat(new File(cache.withSuffix("-name.txt").getRemote()), anExistingFile()); + // Clear the cache. TODO: Would be more realistic to set up security and use WebClient. + ExtensionList.lookupSingleton(LibraryCachingConfiguration.DescriptorImpl.class).doClearCache("library", false); + assertThat(new File(cache.getRemote()), not(anExistingDirectory())); + assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile())); } - //Test similar substrings in "Versions to include" & "Versions to exclude" - //Exclusion takes precedence @Test public void clearCacheConflict() throws Exception { sampleRepo.init(); @@ -314,8 +255,8 @@ public void clearCacheConflict() throws Exception { LibrariesAction action = b.getAction(LibrariesAction.class); LibraryRecord record = action.getLibraries().get(0); FilePath cache = LibraryCachingConfiguration.getGlobalLibrariesCacheDir().child(record.getDirectoryName()); - // Cache should not get created since the version is included in "Versions to exclude" assertThat(new File(cache.getRemote()), not(anExistingDirectory())); assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile())); } + } diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryStepTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryStepTest.java index 7f3b68fb..163c8b14 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryStepTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryStepTest.java @@ -33,8 +33,6 @@ import hudson.plugins.git.SubmoduleConfig; import hudson.plugins.git.UserRemoteConfig; import hudson.plugins.git.extensions.GitSCMExtension; - -import java.io.File; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -101,15 +99,14 @@ public class LibraryStepTest { r.assertLogContains("ran library", b); LibrariesAction action = b.getAction(LibrariesAction.class); assertNotNull(action); - String directoryName = LibraryRecord.directoryNameFor("stuff", String.valueOf(true), GlobalLibraries.ForJob.class.getName()) + File.separator + LibraryRecord.directoryNameFor("master"); - + String directoryName = LibraryRecord.directoryNameFor("stuff", "master", String.valueOf(true), GlobalLibraries.ForJob.class.getName()); assertEquals("[LibraryRecord{name=stuff, version=master, variables=[x], trusted=true, changelog=true, cachingConfiguration=null, directoryName=" + directoryName + "}]", action.getLibraries().toString()); p.setDefinition(new CpsFlowDefinition("library identifier: 'otherstuff@master', retriever: modernSCM([$class: 'GitSCMSource', remote: $/" + sampleRepo + "/$, credentialsId: '']), changelog: false; x()", true)); b = r.buildAndAssertSuccess(p); r.assertLogContains("ran library", b); action = b.getAction(LibrariesAction.class); assertNotNull(action); - directoryName = LibraryRecord.directoryNameFor("otherstuff", String.valueOf(false), LibraryStep.class.getName() + " " + b.getExternalizableId()) + File.separator + LibraryRecord.directoryNameFor("master"); + directoryName = LibraryRecord.directoryNameFor("otherstuff", "master", String.valueOf(false), LibraryStep.class.getName() + " " + b.getExternalizableId()); assertEquals("[LibraryRecord{name=otherstuff, version=master, variables=[x], trusted=false, changelog=false, cachingConfiguration=null, directoryName=" + directoryName + "}]", action.getLibraries().toString()); } diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java index 27e47207..23d2b431 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java @@ -71,7 +71,7 @@ public class ResourceStepTest { initFixedContentLibrary(); LibraryConfiguration libraryConfig = new LibraryConfiguration("stuff", new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); - libraryConfig.setCachingConfiguration(new LibraryCachingConfiguration(0, "", "")); + libraryConfig.setCachingConfiguration(new LibraryCachingConfiguration(0, "","")); GlobalLibraries.get().setLibraries(Collections.singletonList(libraryConfig)); WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); @@ -92,7 +92,7 @@ public class ResourceStepTest { initFixedContentLibrary(); LibraryConfiguration libraryConfig = new LibraryConfiguration("stuff", new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); - libraryConfig.setCachingConfiguration(new LibraryCachingConfiguration(0, "test_unused other", "")); + libraryConfig.setCachingConfiguration(new LibraryCachingConfiguration(0, "test_unused other", null)); GlobalLibraries.get().setLibraries(Collections.singletonList(libraryConfig)); WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); @@ -195,7 +195,7 @@ public class ResourceStepTest { Path resourcesDir = Paths.get(sampleRepo.getRoot().getPath(), "resources"); Files.createDirectories(resourcesDir); Path symlinkPath = Paths.get(resourcesDir.toString(), "master.key"); - Files.createSymbolicLink(symlinkPath, Paths.get("../../../../../../../../secrets/master.key")); + Files.createSymbolicLink(symlinkPath, Paths.get("../../../../../../../secrets/master.key")); sampleRepo.git("add", "src", "resources"); sampleRepo.git("commit", "--message=init"); @@ -270,9 +270,8 @@ public void clearCache(String name) throws Exception { } public void modifyCacheTimestamp(String name, String version, long timestamp) throws Exception { - String cacheDirName = LibraryRecord.directoryNameFor(name, String.valueOf(true), GlobalLibraries.ForJob.class.getName()); - FilePath libraryDir = new FilePath(LibraryCachingConfiguration.getGlobalLibrariesCacheDir(), cacheDirName); - FilePath cacheDir = new FilePath(libraryDir, LibraryRecord.directoryNameFor(version)); + String cacheDirName = LibraryRecord.directoryNameFor(name, version, String.valueOf(true), GlobalLibraries.ForJob.class.getName()); + FilePath cacheDir = new FilePath(LibraryCachingConfiguration.getGlobalLibrariesCacheDir(), cacheDirName); if (cacheDir.exists()) { cacheDir.touch(timestamp); } From a0d1dfe2fb98b4f1ff7b974f6574c7bec87d5f6b Mon Sep 17 00:00:00 2001 From: "KINAXIS\\agharat" Date: Tue, 16 Aug 2022 12:17:24 -0400 Subject: [PATCH 4/8] Modified caching configuration, unit tests --- .../libs/LibraryCachingConfiguration.java | 33 ------------------- .../libs/LibraryCachingConfigurationTest.java | 15 +++++++++ 2 files changed, 15 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java index 944033a7..1037cb7b 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java @@ -158,37 +158,4 @@ public FormValidation doClearCache(@QueryParameter String name, @QueryParameter return FormValidation.ok("The cache dir was deleted successfully."); } } - - /** - * Method can be called from an external source to delete cache of a particular version of the shared library - * Example: delete cache from through pipeline script once the build is successful - * @param jenkinsLibrary Name of the shared library - * @param version Name of the version to delete the cache - * @throws IOException - * @throws InterruptedException - */ - public static void deleteLibraryCacheForBranch(String jenkinsLibrary, String version) throws IOException, InterruptedException { - if (LibraryCachingConfiguration.getGlobalLibrariesCacheDir().exists()) { - for (FilePath libraryNamePath : LibraryCachingConfiguration.getGlobalLibrariesCacheDir().list("*-name.txt")) { - String cacheName; - try (InputStream stream = libraryNamePath.read()) { - cacheName = IOUtils.toString(stream, StandardCharsets.UTF_8); - - if (cacheName.equals(jenkinsLibrary)) { - FilePath libraryCachePath = LibraryCachingConfiguration.getGlobalLibrariesCacheDir() - .child(libraryNamePath.getName().replace("-name.txt", "")); - ReentrantReadWriteLock retrieveLock = LibraryAdder.getReadWriteLockFor(libraryCachePath.getName()); - if (retrieveLock.writeLock().tryLock(10, TimeUnit.SECONDS)) { - try { - libraryCachePath.deleteRecursive(); - libraryNamePath.delete(); - } finally { - retrieveLock.writeLock().unlock(); - } - } - } - } - } - } - } } \ No newline at end of file diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java index 95fe8d42..e3ecd635 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java @@ -190,6 +190,7 @@ public void isExcluded() { assertFalse(substringVersionConfig.isExcluded(null)); } + @Issue("JENKINS-69135") //"Versions to include" feature for caching @Test @WithoutJenkins public void isIncluded() { @@ -205,6 +206,16 @@ public void isIncluded() { assertTrue(substringVersionConfig.isIncluded(SUBSTRING_INCLUDED_VERSIONS_1)); assertTrue(substringVersionConfig.isIncluded(SUBSTRING_INCLUDED_VERSIONS_2)); + assertFalse(nullVersionConfig.isIncluded("")); + assertFalse(oneVersionConfig.isIncluded("")); + assertFalse(multiVersionConfig.isIncluded("")); + assertFalse(substringVersionConfig.isIncluded("")); + + assertFalse(nullVersionConfig.isIncluded(null)); + assertFalse(oneVersionConfig.isIncluded(null)); + assertFalse(multiVersionConfig.isIncluded(null)); + assertFalse(substringVersionConfig.isIncluded(null)); + } @Test @@ -234,6 +245,9 @@ public void clearCache() throws Exception { assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile())); } + //Test similar substrings in "Versions to include" & "Versions to exclude" + //Exclusion takes precedence + @Issue("JENKINS-69135") //"Versions to include" feature for caching @Test public void clearCacheConflict() throws Exception { sampleRepo.init(); @@ -255,6 +269,7 @@ public void clearCacheConflict() throws Exception { LibrariesAction action = b.getAction(LibrariesAction.class); LibraryRecord record = action.getLibraries().get(0); FilePath cache = LibraryCachingConfiguration.getGlobalLibrariesCacheDir().child(record.getDirectoryName()); + // Cache should not get created since the version is included in "Versions to exclude" assertThat(new File(cache.getRemote()), not(anExistingDirectory())); assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile())); } From e5374aa798a8ce090cc85b8f50f7e7ba1f524b55 Mon Sep 17 00:00:00 2001 From: "KINAXIS\\agharat" Date: Thu, 18 Aug 2022 16:16:40 -0400 Subject: [PATCH 5/8] Handling case for empty versions to include --- .../plugins/workflow/libs/LibraryAdder.java | 20 +++++--- .../libs/LibraryCachingConfiguration.java | 7 ++- .../libs/LibraryCachingConfigurationTest.java | 51 +++++++++++++++---- .../workflow/libs/ResourceStepTest.java | 2 +- 4 files changed, 59 insertions(+), 21 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryAdder.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryAdder.java index 1b7b879a..a57ba1e6 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryAdder.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryAdder.java @@ -52,6 +52,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; import java.util.regex.Pattern; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; import org.jenkinsci.plugins.workflow.cps.CpsFlowExecution; import org.jenkinsci.plugins.workflow.cps.GlobalVariable; import org.jenkinsci.plugins.workflow.cps.GlobalVariableSet; @@ -210,7 +211,10 @@ static List retrieve(@NonNull LibraryRecord record, @NonNull LibraryRetriev shouldCache = false; } - if(shouldCache && cachingConfiguration.isIncluded(version)) { + //If the included versions is blank/null, cache irrespective + //else check if that version is included and then cache only that version + + if((shouldCache && cachingConfiguration.isIncluded(version)) || (shouldCache && StringUtils.isBlank(cachingConfiguration.getIncludedVersionsStr()))) { retrieveLock.readLock().lockInterruptibly(); try { CacheStatus cacheStatus = getCacheStatus(cachingConfiguration, versionCacheDir); @@ -220,8 +224,8 @@ static List retrieve(@NonNull LibraryRecord record, @NonNull LibraryRetriev try { boolean retrieve = false; switch (getCacheStatus(cachingConfiguration, versionCacheDir)) { - case VALID: - listener.getLogger().println("Library " + name + "@" + version + " is cached. Copying from home."); + case VALID: + listener.getLogger().println("Library " + name + "@" + version + " is cached. Copying from home."); break; case DOES_NOT_EXIST: retrieve = true; @@ -236,20 +240,20 @@ static List retrieve(@NonNull LibraryRecord record, @NonNull LibraryRetriev retrieve = true; break; } - + if (retrieve) { - listener.getLogger().println("Caching library " + name + "@" + version); + listener.getLogger().println("Caching library " + name + "@" + version); versionCacheDir.mkdirs(); retriever.retrieve(name, version, changelog, versionCacheDir, run, listener); } retrieveLock.readLock().lock(); } finally { - retrieveLock.writeLock().unlock(); + retrieveLock.writeLock().unlock(); } } else { - listener.getLogger().println("Library " + name + "@" + version + " is cached. Copying from home."); + listener.getLogger().println("Library " + name + "@" + version + " is cached. Copying from home."); } - + lastReadFile.touch(System.currentTimeMillis()); versionCacheDir.withSuffix("-name.txt").write(name, "UTF-8"); versionCacheDir.copyRecursiveTo(libDir); diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java index 1037cb7b..efb5484e 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java @@ -57,7 +57,12 @@ public Boolean isRefreshEnabled() { public String getExcludedVersionsStr() { return excludedVersionsStr; } - public String getIncludedVersionsStr() { return includedVersionsStr; } + public String getIncludedVersionsStr() { + if(StringUtils.isBlank(includedVersionsStr)){ + return null; + } + return includedVersionsStr; + } private List getExcludedVersions() { diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java index e3ecd635..7695fa55 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java @@ -206,16 +206,6 @@ public void isIncluded() { assertTrue(substringVersionConfig.isIncluded(SUBSTRING_INCLUDED_VERSIONS_1)); assertTrue(substringVersionConfig.isIncluded(SUBSTRING_INCLUDED_VERSIONS_2)); - assertFalse(nullVersionConfig.isIncluded("")); - assertFalse(oneVersionConfig.isIncluded("")); - assertFalse(multiVersionConfig.isIncluded("")); - assertFalse(substringVersionConfig.isIncluded("")); - - assertFalse(nullVersionConfig.isIncluded(null)); - assertFalse(oneVersionConfig.isIncluded(null)); - assertFalse(multiVersionConfig.isIncluded(null)); - assertFalse(substringVersionConfig.isIncluded(null)); - } @Test @@ -228,7 +218,7 @@ public void clearCache() throws Exception { new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); config.setDefaultVersion("master"); config.setImplicit(true); - config.setCachingConfiguration(new LibraryCachingConfiguration(30, null, "master")); + config.setCachingConfiguration(new LibraryCachingConfiguration(30, null, null)); GlobalLibraries.get().getLibraries().add(config); // Run build and check that cache gets created. WorkflowJob p = r.createProject(WorkflowJob.class); @@ -274,4 +264,43 @@ public void clearCacheConflict() throws Exception { assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile())); } + @Issue("JENKINS-69135") //"Versions to include" feature for caching + @Test + public void clearCacheIncludedVersion() throws Exception { + sampleRepo.init(); + sampleRepo.write("vars/foo.groovy", "def call() { echo 'foo' }"); + sampleRepo.git("add", "vars"); + sampleRepo.git("commit", "--message=init"); + sampleRepo.git("branch", "test/include"); + LibraryConfiguration config = new LibraryConfiguration("library", + new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); + config.setDefaultVersion("master"); + config.setAllowVersionOverride(true); + config.setImplicit(false); + config.setCachingConfiguration(new LibraryCachingConfiguration(30, "", "test/include")); + GlobalLibraries.get().getLibraries().add(config); + // Run build and check that cache gets created. + WorkflowJob p = r.createProject(WorkflowJob.class); + p.setDefinition(new CpsFlowDefinition("library identifier: 'library', changelog:false\n\nfoo()", true)); + WorkflowRun b = r.buildAndAssertSuccess(p); + WorkflowJob p2 = r.createProject(WorkflowJob.class); + p2.setDefinition(new CpsFlowDefinition("library identifier: 'library@test/include', changelog:false\n\nfoo()", true)); + WorkflowRun b2 = r.buildAndAssertSuccess(p2); + LibrariesAction action = b.getAction(LibrariesAction.class); + LibraryRecord record = action.getLibraries().get(0); + LibrariesAction action2 = b2.getAction(LibrariesAction.class); + LibraryRecord record2 = action2.getLibraries().get(0); + FilePath cache = LibraryCachingConfiguration.getGlobalLibrariesCacheDir().child(record.getDirectoryName()); + FilePath cache2 = LibraryCachingConfiguration.getGlobalLibrariesCacheDir().child(record2.getDirectoryName()); + assertThat(new File(cache.getRemote()), not(anExistingDirectory())); + assertThat(new File(cache.withSuffix("-name.txt").getRemote()), not(anExistingFile())); + assertThat(new File(cache2.getRemote()), anExistingDirectory()); + assertThat(new File(cache2.withSuffix("-name.txt").getRemote()), anExistingFile()); + // Clears cache for the entire library, until the "Delete specific cache version" feature in merged + // Clear the cache. TODO: Would be more realistic to set up security and use WebClient. + ExtensionList.lookupSingleton(LibraryCachingConfiguration.DescriptorImpl.class).doClearCache("library", false); + assertThat(new File(cache2.getRemote()), not(anExistingDirectory())); + assertThat(new File(cache2.withSuffix("-name.txt").getRemote()), not(anExistingFile())); + } + } diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java index 23d2b431..aa5ede1f 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java @@ -92,7 +92,7 @@ public class ResourceStepTest { initFixedContentLibrary(); LibraryConfiguration libraryConfig = new LibraryConfiguration("stuff", new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); - libraryConfig.setCachingConfiguration(new LibraryCachingConfiguration(0, "test_unused other", null)); + libraryConfig.setCachingConfiguration(new LibraryCachingConfiguration(0, "test_unused other", "")); GlobalLibraries.get().setLibraries(Collections.singletonList(libraryConfig)); WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); From e0a243267abfdd351c1610b68985c11cea7eb8be Mon Sep 17 00:00:00 2001 From: "KINAXIS\\agharat" Date: Wed, 28 Sep 2022 11:38:40 -0400 Subject: [PATCH 6/8] Refactoring for indentations --- .../org/jenkinsci/plugins/workflow/libs/LibraryAdderTest.java | 2 +- .../org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryAdderTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryAdderTest.java index 3ef953b8..b878e429 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryAdderTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryAdderTest.java @@ -493,7 +493,7 @@ public void parallelBuildsDontInterfereWithExpiredCache() throws Throwable { new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); config.setDefaultVersion("master"); config.setImplicit(true); - config.setCachingConfiguration(new LibraryCachingConfiguration(30, null,null)); + config.setCachingConfiguration(new LibraryCachingConfiguration(30, null, null)); GlobalLibraries.get().getLibraries().add(config); WorkflowJob p1 = r.createProject(WorkflowJob.class); WorkflowJob p2 = r.createProject(WorkflowJob.class); diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java index aa5ede1f..750fde4b 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java @@ -71,7 +71,7 @@ public class ResourceStepTest { initFixedContentLibrary(); LibraryConfiguration libraryConfig = new LibraryConfiguration("stuff", new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); - libraryConfig.setCachingConfiguration(new LibraryCachingConfiguration(0, "","")); + libraryConfig.setCachingConfiguration(new LibraryCachingConfiguration(0, "", "")); GlobalLibraries.get().setLibraries(Collections.singletonList(libraryConfig)); WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); From b1b6712f48acac2a455072a2470f815705745079 Mon Sep 17 00:00:00 2001 From: "KINAXIS\\agharat" Date: Wed, 28 Sep 2022 13:30:50 -0400 Subject: [PATCH 7/8] Updated comments for include versions --- .../plugins/workflow/libs/LibraryCachingConfiguration.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java index efb5484e..ab608b6e 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java @@ -101,8 +101,8 @@ public Boolean isIncluded(String version) { return false; } for (String it : getIncludedVersions()) { - // confirm that the excluded versions aren't null or empty - // and if the version contains the exclusion thus it can be + // works on empty or null included versions + // and if the version contains the inclusion thus it can be // anywhere in the string. if (StringUtils.isNotBlank(it) && version.contains(it)){ return true; From 4d79950ee2337cb466c070db62ef30d72789cd36 Mon Sep 17 00:00:00 2001 From: rsandell Date: Tue, 29 Aug 2023 11:58:52 +0200 Subject: [PATCH 8/8] [JENKINS-69135] Use DataboundSetter --- .../libs/LibraryCachingConfiguration.java | 21 +++++++++++++++++-- .../workflow/libs/GlobalLibrariesTest.java | 10 +++++++++ .../workflow/libs/LibraryAdderTest.java | 6 +++--- .../libs/LibraryCachingCleanupTest.java | 2 +- .../libs/LibraryCachingConfigurationTest.java | 2 +- .../workflow/libs/ResourceStepTest.java | 6 +++--- 6 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java index ab608b6e..3234edb8 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfiguration.java @@ -2,11 +2,15 @@ import hudson.Extension; import hudson.FilePath; +import hudson.RestrictedSince; import hudson.model.AbstractDescribableImpl; import hudson.model.Descriptor; import hudson.util.FormValidation; import jenkins.model.Jenkins; +import org.kohsuke.accmod.Restricted; +import org.kohsuke.accmod.restrictions.NoExternalUse; import org.kohsuke.stapler.DataBoundConstructor; +import org.kohsuke.stapler.DataBoundSetter; import org.kohsuke.stapler.QueryParameter; import java.io.IOException; @@ -35,12 +39,21 @@ public final class LibraryCachingConfiguration extends AbstractDescribableImpl getExcludedVersions() { if (excludedVersionsStr == null) { diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/GlobalLibrariesTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/GlobalLibrariesTest.java index 61da2492..efadbe5d 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/GlobalLibrariesTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/GlobalLibrariesTest.java @@ -57,6 +57,8 @@ public class GlobalLibrariesTest { assertEquals(Collections.emptyList(), gl.getLibraries()); LibraryConfiguration foo = new LibraryConfiguration("foo", new SCMSourceRetriever(new SubversionSCMSource("foo", "https://phony.jenkins.io/foo/"))); LibraryConfiguration bar = new LibraryConfiguration("bar", new SCMSourceRetriever(new GitSCMSource(null, "https://phony.jenkins.io/bar.git", "", "origin", "+refs/heads/*:refs/remotes/origin/*", "*", "", true))); + LibraryCachingConfiguration cachingConfiguration = new LibraryCachingConfiguration(120, "develop", "master stable"); + foo.setCachingConfiguration(cachingConfiguration); bar.setDefaultVersion("master"); bar.setImplicit(true); bar.setAllowVersionOverride(false); @@ -72,6 +74,14 @@ public class GlobalLibrariesTest { r.assertEqualDataBoundBeans(Arrays.asList(foo, bar), libs); libs = gl.getLibraries(); r.assertEqualDataBoundBeans(Arrays.asList(foo, bar), libs); + boolean noFoo = true; + for (LibraryConfiguration lib : libs) { + if ("foo".equals(lib.getName())) { + noFoo = false; + r.assertEqualDataBoundBeans(lib.getCachingConfiguration(), cachingConfiguration); + } + } + assertFalse("Missing a library called foo (should not happen)", noFoo); } @Issue("SECURITY-1422") diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryAdderTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryAdderTest.java index b878e429..2cd590b8 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryAdderTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryAdderTest.java @@ -420,7 +420,7 @@ public class LibraryAdderTest { new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); globalLib.setDefaultVersion("master"); globalLib.setImplicit(true); - globalLib.setCachingConfiguration(new LibraryCachingConfiguration(60, "", "")); + globalLib.setCachingConfiguration(new LibraryCachingConfiguration(60, "")); GlobalLibraries.get().setLibraries(Collections.singletonList(globalLib)); // Create a folder library with the same name and which is also set up to enable caching. sampleRepo2.write("vars/folderLibVar.groovy", "def call() { jenkins.model.Jenkins.get().setSystemMessage('folder library') }"); @@ -430,7 +430,7 @@ public class LibraryAdderTest { new SCMSourceRetriever(new GitSCMSource(null, sampleRepo2.toString(), "", "*", "", true))); folderLib.setDefaultVersion("master"); folderLib.setImplicit(true); - folderLib.setCachingConfiguration(new LibraryCachingConfiguration(60, "", "")); + folderLib.setCachingConfiguration(new LibraryCachingConfiguration(60, "")); Folder f = r.jenkins.createProject(Folder.class, "folder1"); f.getProperties().add(new FolderLibraries(Collections.singletonList(folderLib))); // Create a job that uses the folder library, which will take precedence over the global library, since they have the same name. @@ -493,7 +493,7 @@ public void parallelBuildsDontInterfereWithExpiredCache() throws Throwable { new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); config.setDefaultVersion("master"); config.setImplicit(true); - config.setCachingConfiguration(new LibraryCachingConfiguration(30, null, null)); + config.setCachingConfiguration(new LibraryCachingConfiguration(30, null)); GlobalLibraries.get().getLibraries().add(config); WorkflowJob p1 = r.createProject(WorkflowJob.class); WorkflowJob p2 = r.createProject(WorkflowJob.class); diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanupTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanupTest.java index 9a66cb71..ff30c86e 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanupTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingCleanupTest.java @@ -59,7 +59,7 @@ public void smokes() throws Throwable { new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); config.setDefaultVersion("master"); config.setImplicit(true); - config.setCachingConfiguration(new LibraryCachingConfiguration(30, null, null)); + config.setCachingConfiguration(new LibraryCachingConfiguration(30, null)); GlobalLibraries.get().getLibraries().add(config); // Run build and check that cache gets created. WorkflowJob p = r.createProject(WorkflowJob.class); diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java index 7695fa55..a3689e27 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/LibraryCachingConfigurationTest.java @@ -218,7 +218,7 @@ public void clearCache() throws Exception { new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); config.setDefaultVersion("master"); config.setImplicit(true); - config.setCachingConfiguration(new LibraryCachingConfiguration(30, null, null)); + config.setCachingConfiguration(new LibraryCachingConfiguration(30, null)); GlobalLibraries.get().getLibraries().add(config); // Run build and check that cache gets created. WorkflowJob p = r.createProject(WorkflowJob.class); diff --git a/src/test/java/org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java b/src/test/java/org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java index 750fde4b..c1faefca 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/libs/ResourceStepTest.java @@ -71,7 +71,7 @@ public class ResourceStepTest { initFixedContentLibrary(); LibraryConfiguration libraryConfig = new LibraryConfiguration("stuff", new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); - libraryConfig.setCachingConfiguration(new LibraryCachingConfiguration(0, "", "")); + libraryConfig.setCachingConfiguration(new LibraryCachingConfiguration(0, "")); GlobalLibraries.get().setLibraries(Collections.singletonList(libraryConfig)); WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); @@ -92,7 +92,7 @@ public class ResourceStepTest { initFixedContentLibrary(); LibraryConfiguration libraryConfig = new LibraryConfiguration("stuff", new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); - libraryConfig.setCachingConfiguration(new LibraryCachingConfiguration(0, "test_unused other", "")); + libraryConfig.setCachingConfiguration(new LibraryCachingConfiguration(0, "test_unused other")); GlobalLibraries.get().setLibraries(Collections.singletonList(libraryConfig)); WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); @@ -114,7 +114,7 @@ public class ResourceStepTest { initFixedContentLibrary(); LibraryConfiguration libraryConfig = new LibraryConfiguration("stuff", new SCMSourceRetriever(new GitSCMSource(null, sampleRepo.toString(), "", "*", "", true))); - libraryConfig.setCachingConfiguration(new LibraryCachingConfiguration(60, "test_unused other", "")); + libraryConfig.setCachingConfiguration(new LibraryCachingConfiguration(60, "test_unused other")); GlobalLibraries.get().setLibraries(Collections.singletonList(libraryConfig)); WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p");