diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java b/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java index 1c9565830f0..aeb05a8073a 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java @@ -27,27 +27,35 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.StringReader; +import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; -import java.nio.charset.StandardCharsets; import java.text.MessageFormat; import java.time.Duration; +import java.time.LocalDate; import java.time.ZoneId; +import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.Properties; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.function.Function; import java.util.zip.GZIPOutputStream; + +import org.jetbrains.annotations.NotNull; import org.owasp.dependencycheck.Engine; import org.owasp.dependencycheck.data.nvdcve.CveDB; import org.owasp.dependencycheck.data.nvdcve.DatabaseException; @@ -59,12 +67,15 @@ import org.owasp.dependencycheck.utils.DownloadFailedException; import org.owasp.dependencycheck.utils.Downloader; import org.owasp.dependencycheck.utils.InvalidSettingException; +import org.owasp.dependencycheck.utils.Pair; import org.owasp.dependencycheck.utils.ResourceNotFoundException; import org.owasp.dependencycheck.utils.Settings; import org.owasp.dependencycheck.utils.TooManyRequestsException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static java.nio.charset.StandardCharsets.UTF_8; + /** * * @author Jeremy Long @@ -117,44 +128,22 @@ public boolean update(Engine engine) throws UpdateException { return processApi(); } - protected UrlData extractUrlData(String nvdDataFeedUrl) { - String url; - String pattern = null; - if (nvdDataFeedUrl.endsWith(".json.gz")) { - final int lio = nvdDataFeedUrl.lastIndexOf("/"); - pattern = nvdDataFeedUrl.substring(lio + 1); - url = nvdDataFeedUrl.substring(0, lio); - } else { - url = nvdDataFeedUrl; - } - if (!url.endsWith("/")) { - url += "/"; - } - return new UrlData(url, pattern); - } - private boolean processDatafeed(String nvdDataFeedUrl) throws UpdateException { boolean updatesMade = false; try { dbProperties = cveDb.getDatabaseProperties(); if (checkUpdate()) { - final UrlData data = extractUrlData(nvdDataFeedUrl); - final String url = data.getUrl(); - String pattern = data.getPattern(); - final Properties cacheProperties = getRemoteCacheProperties(url, pattern); - if (pattern == null) { - final String prefix = cacheProperties.getProperty("prefix", "nvdcve-"); - pattern = prefix + "{0}.json.gz"; - } + FeedUrl urlData = FeedUrl.extractFromUrlOptionalPattern(nvdDataFeedUrl); + final Properties cacheProperties = getRemoteDataFeedCacheProperties(urlData); + urlData = urlData.withPattern(p -> p.orElse(cacheProperties.getProperty("prefix", FeedUrl.DEFAULT_FILE_PATTERN_PREFIX) + FeedUrl.DEFAULT_FILE_PATTERN_SUFFIX)); - final ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC")); - final Map updateable = getUpdatesNeeded(url, pattern, cacheProperties, now); + final ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); + final Map updateable = getUpdatesNeeded(urlData, cacheProperties, now); if (!updateable.isEmpty()) { final int max = settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 1); final int downloadPoolSize = Math.min(Runtime.getRuntime().availableProcessors(), max); // going over 2 threads does not appear to improve performance - final int maxExec = PROCESSING_THREAD_POOL_SIZE; - final int execPoolSize = Math.min(maxExec, 2); + final int execPoolSize = Math.min(PROCESSING_THREAD_POOL_SIZE, 2); ExecutorService processingExecutorService = null; ExecutorService downloadExecutorService = null; @@ -530,29 +519,24 @@ private boolean dataExists() { * be refreshed this method will return the NvdCveUrl for the files that * need to be updated. * - * @param url the URL of the NVD API cache - * @param filePattern the string format pattern for the cached files (e.g. - * "nvdcve-{0}.json.gz") - * @param cacheProperties the properties from the remote NVD API cache + * @param feedUrl a parsed NVD cache / data feed URL + * @param cacheProperties the properties from the remote NVD API cache or data feed * @param now the start time of the update process * @return the map of key to URLs - where the key is the year or `modified` * @throws UpdateException Is thrown if there is an issue with the last * updated properties file */ - protected final Map getUpdatesNeeded(String url, String filePattern, - Properties cacheProperties, ZonedDateTime now) throws UpdateException { + protected final Map getUpdatesNeeded(FeedUrl feedUrl, Properties cacheProperties, ZonedDateTime now) throws UpdateException { LOGGER.debug("starting getUpdatesNeeded() ..."); final Map updates = new HashMap<>(); if (dbProperties != null && !dbProperties.isEmpty()) { - final int startYear = settings.getInt(Settings.KEYS.NVD_API_DATAFEED_START_YEAR, 2002); - // for establishing the current year use the timezone where the new year starts first - // as from that moment on CNAs might start assigning CVEs with the new year depending - // on the CNA's timezone - final int endYear = now.withZoneSameInstant(ZoneId.of("UTC+14:00")).getYear(); + Pair yearRange = FeedUrl.toYearRange(settings, now); + int startYear = yearRange.getLeft(); + int endYear = yearRange.getRight(); boolean needsFullUpdate = false; for (int y = startYear; y <= endYear; y++) { final ZonedDateTime val = dbProperties.getTimestamp(DatabaseProperties.NVD_CACHE_LAST_MODIFIED + "." + y); - if (val == null) { + if (val == null && FeedUrl.isMandatoryFeedYear(now, y)) { needsFullUpdate = true; break; } @@ -563,11 +547,11 @@ protected final Map getUpdatesNeeded(String url, String filePatt if (!needsFullUpdate && lastUpdated.equals(DatabaseProperties.getTimestamp(cacheProperties, NVD_API_CACHE_MODIFIED_DATE))) { return updates; } else { - updates.put("modified", url + MessageFormat.format(filePattern, "modified")); + updates.put("modified", feedUrl.toFormattedUrlString("modified")); if (needsFullUpdate) { for (int i = startYear; i <= endYear; i++) { if (cacheProperties.containsKey(NVD_API_CACHE_MODIFIED_DATE + "." + i)) { - updates.put(String.valueOf(i), url + MessageFormat.format(filePattern, String.valueOf(i))); + updates.put(String.valueOf(i), feedUrl.toFormattedUrlString(i)); } } } else if (!DateUtil.withinDateRange(lastUpdated, now, days)) { @@ -576,8 +560,8 @@ protected final Map getUpdatesNeeded(String url, String filePatt final ZonedDateTime lastModifiedCache = DatabaseProperties.getTimestamp(cacheProperties, NVD_API_CACHE_MODIFIED_DATE + "." + i); final ZonedDateTime lastModifiedDB = dbProperties.getTimestamp(DatabaseProperties.NVD_CACHE_LAST_MODIFIED + "." + i); - if (lastModifiedDB == null || lastModifiedCache.compareTo(lastModifiedDB) > 0) { - updates.put(String.valueOf(i), url + MessageFormat.format(filePattern, String.valueOf(i))); + if (lastModifiedDB == null || (lastModifiedCache != null && lastModifiedCache.compareTo(lastModifiedDB) > 0)) { + updates.put(String.valueOf(i), feedUrl.toFormattedUrlString(i)); } } } @@ -585,70 +569,86 @@ protected final Map getUpdatesNeeded(String url, String filePatt } } if (updates.size() > 3) { - LOGGER.info("NVD API Cache requires several updates; this could take a couple of minutes."); + LOGGER.info("NVD API Cache / Data Feed requires several updates; this could take a couple of minutes."); } return updates; } /** - * Downloads the metadata properties of the NVD API cache. + * Downloads the metadata properties of the NVD API cache / data feed. * - * @param url the base URL to the NVD API cache - * @param pattern the pattern of the datafile name for the NVD API cache + * @param dataFeedUrl a parsed NVD cache / data feed URL * @return the cache properties - * @throws UpdateException thrown if the properties file could not be - * downloaded + * @throws UpdateException thrown if the properties file could not be downloaded */ - protected final Properties getRemoteCacheProperties(String url, String pattern) throws UpdateException { - final Properties properties = new Properties(); + protected final Properties getRemoteDataFeedCacheProperties(FeedUrl dataFeedUrl) throws UpdateException { try { - final URL u = new URI(url + "cache.properties").toURL(); - final String content = Downloader.getInstance().fetchContent(u, StandardCharsets.UTF_8); + final Properties properties = new Properties(); + final String content = Downloader.getInstance().fetchContent(dataFeedUrl.toSuffixedUrl("cache.properties"), UTF_8); properties.load(new StringReader(content)); + return properties; - } catch (URISyntaxException ex) { - throw new UpdateException("Invalid NVD Cache URL", ex); } catch (DownloadFailedException | ResourceNotFoundException ex) { - final String metaPattern; - if (pattern == null) { - metaPattern = "nvdcve-{0}.meta"; - } else { - metaPattern = pattern.replace(".json.gz", ".meta"); - } - try { - URL metaUrl = new URI(url + MessageFormat.format(metaPattern, "modified")).toURL(); - String content = Downloader.getInstance().fetchContent(metaUrl, StandardCharsets.UTF_8); - final Properties props = new Properties(); - props.load(new StringReader(content)); - ZonedDateTime lmd = DatabaseProperties.getIsoTimestamp(props, "lastModifiedDate"); - DatabaseProperties.setTimestamp(properties, "lastModifiedDate.modified", lmd); - DatabaseProperties.setTimestamp(properties, "lastModifiedDate", lmd); - final int startYear = settings.getInt(Settings.KEYS.NVD_API_DATAFEED_START_YEAR, 2002); - final ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC")); - final int endYear = now.withZoneSameInstant(ZoneId.of("UTC+14:00")).getYear(); - for (int y = startYear; y <= endYear; y++) { - metaUrl = new URI(url + MessageFormat.format(metaPattern, String.valueOf(y))).toURL(); - content = Downloader.getInstance().fetchContent(metaUrl, StandardCharsets.UTF_8); - props.clear(); - props.load(new StringReader(content)); - lmd = DatabaseProperties.getIsoTimestamp(props, "lastModifiedDate"); - DatabaseProperties.setTimestamp(properties, "lastModifiedDate." + String.valueOf(y), lmd); - } - } catch (URISyntaxException | TooManyRequestsException | ResourceNotFoundException | IOException ex1) { - throw new UpdateException("Unable to download the data feed META files", ex); - } + LOGGER.debug("Unable to download the NVD API cache.properties due to [{}]; attempting to build from data feed metadata files instead...", ex.toString()); + return generateRemoteDataFeedCachePropertiesFromMetadata(dataFeedUrl); + } catch (URISyntaxException | MalformedURLException ex) { + throw new UpdateException("Invalid NVD Cache / Data Feed URL", ex); } catch (TooManyRequestsException ex) { throw new UpdateException("Unable to download the NVD API cache.properties", ex); } catch (IOException ex) { - throw new UpdateException("Invalid NVD Cache Properties file contents", ex); + throw new UpdateException("Invalid NVD Cache properties file contents", ex); } + } + + /** + * Builds the metadata properties from individual metadata fields within the data feed + * + * @param dataFeedUrl a parsed NVD cache / data feed URL + * @return the cache properties + * @throws UpdateException thrown if the metadata files could not be downloaded to build cache properties + */ + private Properties generateRemoteDataFeedCachePropertiesFromMetadata(FeedUrl dataFeedUrl) throws UpdateException { + FeedUrl metaFeedUrl = dataFeedUrl.withPattern(p -> p + .orElse(FeedUrl.DEFAULT_FILE_PATTERN) + .replace(".json.gz", ".meta") + ); + + final Properties properties = new Properties(); + ZonedDateTime lmd = metaFeedUrl.getLastModifiedFor("modified"); + DatabaseProperties.setTimestamp(properties, NVD_API_CACHE_MODIFIED_DATE + ".modified", lmd); + DatabaseProperties.setTimestamp(properties, NVD_API_CACHE_MODIFIED_DATE, lmd); + + metaFeedUrl.getLastModifiedDatePropertiesByYear(this.settings, ZonedDateTime.now(ZoneOffset.UTC)) + .forEach((k, v) -> DatabaseProperties.setTimestamp(properties, k, v)); return properties; } - protected static class UrlData { + protected static class FeedUrl { /** - * The URL to download resources from. + * Default file pattern prefix for NVD caches; generally those generated by vulnz / Open Vulnerability Clients + */ + static final String DEFAULT_FILE_PATTERN_PREFIX = "nvdcve-"; + /** + * Default file pattern suffix for NVD caches; generally those generated by vulnz / Open Vulnerability Clients + */ + static final String DEFAULT_FILE_PATTERN_SUFFIX = "{0}.json.gz"; + /** + * Default file pattern for NVD caches; generally those generated by vulnz / Open Vulnerability Clients + */ + static final String DEFAULT_FILE_PATTERN = DEFAULT_FILE_PATTERN_PREFIX + DEFAULT_FILE_PATTERN_SUFFIX; + + /** + * The timezone where the new year starts first. + */ + static final ZoneId ZONE_GLOBAL_EARLIEST = ZoneId.of("UTC+14:00"); + /** + * The timezone where the new year starts last. + */ + static final ZoneId ZONE_GLOBAL_LATEST = ZoneId.of("UTC-12:00"); + + /** + * The base URL to download resources from. */ private final String url; @@ -657,28 +657,109 @@ protected static class UrlData { */ private final String pattern; - public UrlData(String url, String pattern) { + public FeedUrl(String url, String pattern) { this.url = url; this.pattern = pattern; } + public FeedUrl withPattern(Function, String> patternTransformer) { + return new FeedUrl(url, patternTransformer.apply(Optional.ofNullable(pattern))); + } + + @NotNull String toFormattedUrlString(String formatArg) { + return url + MessageFormat.format(Optional.ofNullable(pattern).orElseThrow(), formatArg); + } + + @NotNull String toFormattedUrlString(int formatArg) { + return toFormattedUrlString(String.valueOf(formatArg)); + } + + @NotNull URL toFormattedUrl(@NotNull String formatArg) throws MalformedURLException, URISyntaxException { + return new URI(toFormattedUrlString(formatArg)).toURL(); + } + + @SuppressWarnings("SameParameterValue") + @NotNull URL toSuffixedUrl(String suffix) throws MalformedURLException, URISyntaxException { + return new URI(url + suffix).toURL(); + } + /** - * Get the value of pattern - * - * @return the value of pattern + * @param url A NVD data feed URL which may be just a base URL such as https://my-nvd-cache/nvd_cache or + * may include a formatted URL ending with .json.gz such as https://nvd.nist.gov/feeds/json/cve/2.0/nvdcve-2.0-{0}.json.gz + * @return A constructed FeedUrl object */ - public String getPattern() { - return pattern; + @SuppressWarnings("JavadocLinkAsPlainText") + protected static FeedUrl extractFromUrlOptionalPattern(String url) { + String baseUrl; + String pattern = null; + if (url.endsWith(".json.gz")) { + final int lio = url.lastIndexOf("/"); + pattern = url.substring(lio + 1); + baseUrl = url.substring(0, lio); + } else { + baseUrl = url; + } + if (!baseUrl.endsWith("/")) { + baseUrl += "/"; + } + return new FeedUrl(baseUrl, pattern); + } + + private static @NotNull Pair toYearRange(Settings settings, ZonedDateTime now) { + // for establishing the current year use the timezone where the new year starts first + // as from that moment on CNAs might start assigning CVEs with the new year depending + // on the CNA's timezone + final int startYear = settings.getInt(Settings.KEYS.NVD_API_DATAFEED_START_YEAR, 2002); + final int endYear = now.withZoneSameInstant(ZONE_GLOBAL_EARLIEST).getYear(); + return new Pair<>(startYear, endYear); + } + + private @NotNull ZonedDateTime getLastModifiedFor(int year) throws UpdateException { + return getLastModifiedFor(String.valueOf(year)); + } + + private @NotNull ZonedDateTime getLastModifiedFor(String fileVersion) throws UpdateException { + try { + String content = Downloader.getInstance().fetchContent(toFormattedUrl(fileVersion), UTF_8); + Properties props = new Properties(); + props.load(new StringReader(content)); + return Objects.requireNonNull(DatabaseProperties.getIsoTimestamp(props, NVD_API_CACHE_MODIFIED_DATE)); + } catch (Exception ex) { + throw new UpdateException("Unable to download & parse the data feed .meta file for " + fileVersion, ex); + } + } + + Map getLastModifiedDatePropertiesByYear(Settings settings, ZonedDateTime now) throws UpdateException { + Pair yearRange = toYearRange(settings, now); + Map lastModifiedDateProperties = new LinkedHashMap<>(); + for (int y = yearRange.getLeft(); y <= yearRange.getRight(); y++) { + try { + lastModifiedDateProperties.put(NVD_API_CACHE_MODIFIED_DATE + "." + y, getLastModifiedFor(y)); + } catch (UpdateException e) { + if (isMandatoryFeedYear(now, y)) { + throw e; + } + LOGGER.debug("Ignoring data feed metadata retrieval failure for {}, it is still January 1st in some TZ; so feed files may not yet be generated. Error was {}", y, e.toString()); + } + } + return lastModifiedDateProperties; } /** - * Get the value of url - * - * @return the value of url + * @param now The current time in any timezone + * @param targetYear Target year's feed data to retrieve + * @return Whether or not the targetYear is considered a mandatory feed file to retrieve given the target year and current time. */ - public String getUrl() { - return url; + static boolean isMandatoryFeedYear(ZonedDateTime now, int targetYear) { + return isNotTargetYearInAnyTZ(now, targetYear) || isAfterJanuary1InEveryTZ(now, targetYear); + } + + private static boolean isNotTargetYearInAnyTZ(ZonedDateTime now, int targetYear) { + return targetYear != now.withZoneSameInstant(ZONE_GLOBAL_EARLIEST).getYear(); } + private static boolean isAfterJanuary1InEveryTZ(ZonedDateTime now, int targetYear) { + return now.isAfter(LocalDate.of(targetYear, 1, 2).atStartOfDay().atZone(ZONE_GLOBAL_LATEST)); + } } } diff --git a/core/src/test/java/org/owasp/dependencycheck/data/update/NvdApiDataSourceTest.java b/core/src/test/java/org/owasp/dependencycheck/data/update/NvdApiDataSourceTest.java index ae73fac49df..b0dd6d38cda 100644 --- a/core/src/test/java/org/owasp/dependencycheck/data/update/NvdApiDataSourceTest.java +++ b/core/src/test/java/org/owasp/dependencycheck/data/update/NvdApiDataSourceTest.java @@ -17,55 +17,205 @@ */ package org.owasp.dependencycheck.data.update; +import org.hamcrest.Matchers; +import org.jspecify.annotations.NonNull; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.owasp.dependencycheck.data.update.exception.UpdateException; +import org.owasp.dependencycheck.utils.DownloadFailedException; +import org.owasp.dependencycheck.utils.Downloader; +import org.owasp.dependencycheck.utils.Settings; +import java.net.URI; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.everyItem; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; +import static org.owasp.dependencycheck.data.update.NvdApiDataSource.FeedUrl.DEFAULT_FILE_PATTERN; +import static org.owasp.dependencycheck.data.update.NvdApiDataSource.FeedUrl.extractFromUrlOptionalPattern; +import static org.owasp.dependencycheck.data.update.NvdApiDataSource.FeedUrl.isMandatoryFeedYear; -/** - * - * @author Jeremy Long - */ class NvdApiDataSourceTest { - /** - * Test of extractUrlData method, of class NvdApiDataSource. - */ - @Test - void testExtractUrlData() { - String nvdDataFeedUrl = "https://internal.server/nist/nvdcve-{0}.json.gz"; - NvdApiDataSource instance = new NvdApiDataSource(); - String expectedUrl = "https://internal.server/nist/"; - String expectedPattern = "nvdcve-{0}.json.gz"; - NvdApiDataSource.UrlData result = instance.extractUrlData(nvdDataFeedUrl); - - nvdDataFeedUrl = "https://internal.server/nist/"; - expectedUrl = "https://internal.server/nist/"; - result = instance.extractUrlData(nvdDataFeedUrl); - - assertEquals(expectedUrl, result.getUrl()); - assertNull(result.getPattern()); - - nvdDataFeedUrl = "https://internal.server/nist"; - expectedUrl = "https://internal.server/nist/"; - result = instance.extractUrlData(nvdDataFeedUrl); - - assertEquals(expectedUrl, result.getUrl()); - assertNull(result.getPattern()); + @Nested + class FeedUrlParsing { + + @Test + void shouldExtractUrlWithPattern() throws Exception { + String nvdDataFeedUrl = "https://internal.server/nist/nvdcve-{0}.json.gz"; + String expectedUrl = "https://internal.server/nist/nvdcve-2045.json.gz"; + NvdApiDataSource.FeedUrl result = extractFromUrlOptionalPattern(nvdDataFeedUrl); + + assertEquals(expectedUrl, result.toFormattedUrlString("2045")); + assertEquals(URI.create(expectedUrl).toURL(), result.toFormattedUrl("2045")); + assertEquals(URI.create("https://internal.server/nist/some-file.txt").toURL(), result.toSuffixedUrl("some-file.txt")); + + assertEquals(expectedUrl, result.toFormattedUrlString("2045")); + assertEquals(URI.create(expectedUrl).toURL(), result.toFormattedUrl("2045")); + } + + @Test + void shouldAllowTransformingFilePattern() { + NvdApiDataSource.FeedUrl result = extractFromUrlOptionalPattern("https://internal.server/nist/nvdcve-{0}.json.gz") + .withPattern(p -> p.orElseThrow().replace(".json.gz", ".something")); + assertEquals("https://internal.server/nist/nvdcve-ok.something", result.toFormattedUrlString("ok")); + + NvdApiDataSource.FeedUrl resultNoPattern = extractFromUrlOptionalPattern("https://internal.server/nist/") + .withPattern(p -> p.orElse("my-suffix-{0}.json.gz")); + assertEquals("https://internal.server/nist/my-suffix-ok.json.gz", resultNoPattern.toFormattedUrlString("ok")); + } + + @Test + void shouldExtractUrlWithoutPattern() throws Exception { + String nvdDataFeedUrl = "https://internal.server/nist/"; + NvdApiDataSource.FeedUrl result = extractFromUrlOptionalPattern(nvdDataFeedUrl); + + assertThrows(NoSuchElementException.class, () -> result.toFormattedUrlString("2045")); + assertThrows(NoSuchElementException.class, () -> result.toFormattedUrl("2045")); + assertEquals(URI.create("https://internal.server/nist/some-file.txt").toURL(), result.toSuffixedUrl("some-file.txt")); + + String expectedUrl = "https://internal.server/nist/nvdcve-2045.json.gz"; + NvdApiDataSource.FeedUrl resultWithPattern = extractFromUrlOptionalPattern(nvdDataFeedUrl) + .withPattern(p -> p.orElse(DEFAULT_FILE_PATTERN)); + + assertEquals(expectedUrl, resultWithPattern.toFormattedUrlString("2045")); + assertEquals(URI.create(expectedUrl).toURL(), resultWithPattern.toFormattedUrl("2045")); + } + + @Test + void extractUrlWithoutPatternShouldAddTrailingSlashes() { + String nvdDataFeedUrl = "https://internal.server/nist"; + String expectedUrl = "https://internal.server/nist/nvdcve-2045.json.gz"; + + NvdApiDataSource.FeedUrl result = extractFromUrlOptionalPattern(nvdDataFeedUrl) + .withPattern(p -> p.orElse(DEFAULT_FILE_PATTERN)); + + assertEquals(expectedUrl, result.toFormattedUrlString("2045")); + } } -// /** -// * Test of getRemoteCacheProperties method, of class NvdApiDataSource. -// */ -// @Test -// public void testGetRemoteCacheProperties() throws Exception { -// System.out.println("getRemoteCacheProperties"); -// String url = ""; -// NvdApiDataSource instance = new NvdApiDataSource(); -// Properties expResult = null; -// Properties result = instance.getRemoteCacheProperties(url); -// assertEquals(expResult, result); -// // TODO review the generated test code and remove the default call to fail. -// fail("The test case is a prototype."); -// } + @Nested + class FeedUrlMandatoryYears { + + @Test + void shouldConsiderYearsMandatoryWhenNotCurrentYearAtEarliestTZ() { + ZonedDateTime janFirst2004AtEarliest = ZonedDateTime.of(2004, 1, 1, 0, 0, 0, 0, NvdApiDataSource.FeedUrl.ZONE_GLOBAL_EARLIEST); + assertTrue(isMandatoryFeedYear(janFirst2004AtEarliest, 2002)); + assertTrue(isMandatoryFeedYear(janFirst2004AtEarliest, 2003)); + assertFalse(isMandatoryFeedYear(janFirst2004AtEarliest, 2004)); + } + + @Test + void shouldConsiderYearsMandatoryWhenNotCurrentYearAtLatestTZ() { + ZonedDateTime janFirst2004AtLatest = ZonedDateTime.of(2004, 1, 1, 0, 0, 0, 0, NvdApiDataSource.FeedUrl.ZONE_GLOBAL_LATEST); + assertTrue(isMandatoryFeedYear(janFirst2004AtLatest, 2002)); + assertTrue(isMandatoryFeedYear(janFirst2004AtLatest, 2003)); + assertFalse(isMandatoryFeedYear(janFirst2004AtLatest, 2004)); + } + + @Test + void shouldConsiderYearsMandatoryWhenNoLongerJan1Anywhere() { + // It's still Jan 1 somewhere... + ZonedDateTime janSecond2004AtEarliest = ZonedDateTime.of(2004, 1, 2, 0, 0, 0, 0, NvdApiDataSource.FeedUrl.ZONE_GLOBAL_EARLIEST); + assertFalse(isMandatoryFeedYear(janSecond2004AtEarliest, 2004)); + + // Until it's no longer Jan 1 anywhere + ZonedDateTime janSecond2004AtLatest = ZonedDateTime.of(2004, 1, 2, 0, 0, 0, 1, NvdApiDataSource.FeedUrl.ZONE_GLOBAL_LATEST); + assertTrue(isMandatoryFeedYear(janSecond2004AtLatest, 2004)); + } + } + + @Nested + class FeedUrlMetadataRetrieval { + + @Test + void shouldRetrieveMetadataByYear() throws Exception { + try (MockedStatic downloaderClass = mockStatic(Downloader.class)) { + Downloader downloader = mock(Downloader.class); + when(downloader.fetchContent(any(), any())).thenReturn("lastModifiedDate=2013-01-01T12:00:00Z"); + downloaderClass.when(Downloader::getInstance).thenReturn(downloader); + + assertThat(retrieveUntil(ZonedDateTime.of(2003, 12, 1, 0, 0, 0, 0, ZoneOffset.UTC)).keySet(), + contains("lastModifiedDate.2002", "lastModifiedDate.2003")); + } + } + + @Test + void shouldRetrieveMetadataForNextYearOnJan1AtEarliestTZ() throws Exception { + try (MockedStatic downloaderClass = mockStatic(Downloader.class)) { + Downloader downloader = mock(Downloader.class); + when(downloader.fetchContent(any(), any())).thenReturn("lastModifiedDate=2013-01-01T12:00:00Z"); + downloaderClass.when(Downloader::getInstance).thenReturn(downloader); + + ZonedDateTime jan1Earliest = ZonedDateTime.of(2004, 1, 1, 0, 0, 0, 0, NvdApiDataSource.FeedUrl.ZONE_GLOBAL_EARLIEST); + assertThat(retrieveUntil(jan1Earliest.minusSeconds(1)).keySet(), + contains("lastModifiedDate.2002", "lastModifiedDate.2003")); + + assertThat(retrieveUntil(jan1Earliest).keySet(), + contains("lastModifiedDate.2002", "lastModifiedDate.2003", "lastModifiedDate.2004")); + + assertThat(retrieveUntil(ZonedDateTime.of(2004, 1, 1, 0, 0, 0, 0, NvdApiDataSource.FeedUrl.ZONE_GLOBAL_LATEST)).keySet(), + contains("lastModifiedDate.2002", "lastModifiedDate.2003", "lastModifiedDate.2004")); + } + } + + @Test + void shouldNormallyRethrowDownloadErrorsEvenIfJan1OnEndYear() throws Exception { + try (MockedStatic downloaderClass = mockStatic(Downloader.class)) { + Downloader downloader = mock(Downloader.class); + when(downloader.fetchContent(any(), any())).thenThrow(new DownloadFailedException("failed to download")); + downloaderClass.when(Downloader::getInstance).thenReturn(downloader); + + assertThrows(UpdateException.class, () -> retrieveUntil(ZonedDateTime.of(2003, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC))); + } + } + + @Test + void shouldIgnoreDownloadFailureForFinalYearIfStillJan1() throws Exception { + List untilDates = List.of( + ZonedDateTime.of(2004, 1, 1, 0, 0, 0, 0, NvdApiDataSource.FeedUrl.ZONE_GLOBAL_EARLIEST), + ZonedDateTime.of(2004, 1, 2, 0, 0, 0, 0, NvdApiDataSource.FeedUrl.ZONE_GLOBAL_LATEST) + .minusSeconds(1) + ); + + for (ZonedDateTime until : untilDates) { + try (MockedStatic downloaderClass = mockStatic(Downloader.class)) { + Downloader downloader = mock(Downloader.class); + when(downloader.fetchContent(any(), any())) + .thenReturn("lastModifiedDate=2013-01-01T12:00:00Z") + .thenReturn("lastModifiedDate=2013-01-01T12:00:00Z") + .thenThrow(new DownloadFailedException("failed to download 3rd file")); + + downloaderClass.when(Downloader::getInstance).thenReturn(downloader); + + assertThat(retrieveUntil(until).keySet(), + contains("lastModifiedDate.2002", "lastModifiedDate.2003")); + } + } + } + + private @NonNull Map retrieveUntil(ZonedDateTime until) throws UpdateException { + Map lastModifieds; + NvdApiDataSource.FeedUrl feedUrl = extractFromUrlOptionalPattern("https://internal.server/nist/nvdcve-{0}.json.gz"); + + lastModifieds = feedUrl.getLastModifiedDatePropertiesByYear(new Settings(), until); + + assertThat(lastModifieds.values(), everyItem(Matchers.equalTo(ZonedDateTime.of(2013, 1, 1, 12, 0, 0, 0, ZoneOffset.UTC)))); + return lastModifieds; + } + } } diff --git a/core/src/test/java/org/owasp/dependencycheck/xml/suppression/SuppressionParserTest.java b/core/src/test/java/org/owasp/dependencycheck/xml/suppression/SuppressionParserTest.java index 8bcc0e9e832..727fac14e67 100644 --- a/core/src/test/java/org/owasp/dependencycheck/xml/suppression/SuppressionParserTest.java +++ b/core/src/test/java/org/owasp/dependencycheck/xml/suppression/SuppressionParserTest.java @@ -131,7 +131,7 @@ void testParseSuppressionV1dot4Inherits() throws SuppressionParseException { assertEquals(1, filteredSuppressions.size()); SuppressionRule rule = filteredSuppressions.get(0); - Instant expectedTime = LocalDate.of(2026, 1, 1) + Instant expectedTime = LocalDate.of(2046, 1, 1) .atStartOfDay(ZoneOffset.UTC) .toInstant(); assertEquals(expectedTime, rule.getUntil().toInstant()); diff --git a/core/src/test/resources/suppressions_1_4.xml b/core/src/test/resources/suppressions_1_4.xml index 652f1a3a8a1..b254bd21736 100644 --- a/core/src/test/resources/suppressions_1_4.xml +++ b/core/src/test/resources/suppressions_1_4.xml @@ -38,7 +38,7 @@ - + Temporary Suppressions to be reexamined later