fix: Don't consider a new year's feed files or metadata mandatory until it is January 2nd everywhere

chadlwilson · chadlwilson · commit 4e9d846f4f8f · 2026-01-02T17:40:16.000+08:00
This avoids failures on January 1st given feed upload times are not guaranteed to be at any particular time of day in any particular timezone. We assume the metadaa and feed should exist once it is January 2nd in the "earliest" TZ on earth.

Signed-off-by: Chad Wilson &lt;29788154+chadlwilson@users.noreply.github.com&gt;
diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java b/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java
@@ -33,12 +33,15 @@
 import java.net.URL;
 import java.text.MessageFormat;
 import java.time.Duration;
+import java.time.LocalDate;
 import java.time.ZoneId;
+import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -52,6 +55,8 @@
 import java.util.zip.GZIPOutputStream;
 
 import org.jetbrains.annotations.NotNull;
+import org.jspecify.annotations.NonNull;
+import org.jspecify.annotations.Nullable;
 import org.owasp.dependencycheck.Engine;
 import org.owasp.dependencycheck.data.nvdcve.CveDB;
 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
@@ -63,6 +68,7 @@
 import org.owasp.dependencycheck.utils.DownloadFailedException;
 import org.owasp.dependencycheck.utils.Downloader;
 import org.owasp.dependencycheck.utils.InvalidSettingException;
+import org.owasp.dependencycheck.utils.Pair;
 import org.owasp.dependencycheck.utils.ResourceNotFoundException;
 import org.owasp.dependencycheck.utils.Settings;
 import org.owasp.dependencycheck.utils.TooManyRequestsException;
@@ -132,7 +138,7 @@ private boolean processDatafeed(String nvdDataFeedUrl) throws UpdateException {
                 final Properties cacheProperties = getRemoteDataFeedCacheProperties(urlData);
                 urlData = urlData.withPattern(p -> p.orElse(cacheProperties.getProperty("prefix", FeedUrl.DEFAULT_FILE_PATTERN_PREFIX) + FeedUrl.DEFAULT_FILE_PATTERN_SUFFIX));
 
-                final ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC"));
+                final ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
                 final Map<String, String> updateable = getUpdatesNeeded(urlData, cacheProperties, now);
                 if (!updateable.isEmpty()) {
                     final int max = settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 1);
@@ -525,15 +531,13 @@ protected final Map<String, String> getUpdatesNeeded(FeedUrl feedUrl, Properties
         LOGGER.debug("starting getUpdatesNeeded() ...");
         final Map<String, String> updates = new HashMap<>();
         if (dbProperties != null && !dbProperties.isEmpty()) {
-            final int startYear = settings.getInt(Settings.KEYS.NVD_API_DATAFEED_START_YEAR, 2002);
-            // for establishing the current year use the timezone where the new year starts first
-            // as from that moment on CNAs might start assigning CVEs with the new year depending
-            // on the CNA's timezone
-            final int endYear = now.withZoneSameInstant(ZoneId.of("UTC+14:00")).getYear();
+            Pair<Integer, Integer> yearRange = FeedUrl.toYearRange(settings, now);
+            int startYear = yearRange.getLeft();
+            int endYear = yearRange.getRight();
             boolean needsFullUpdate = false;
             for (int y = startYear; y <= endYear; y++) {
                 final ZonedDateTime val = dbProperties.getTimestamp(DatabaseProperties.NVD_CACHE_LAST_MODIFIED + "." + y);
-                if (val == null) {
+                if (val == null && FeedUrl.isMandatoryFeedYear(now, y)) {
                     needsFullUpdate = true;
                     break;
                 }
@@ -611,27 +615,13 @@ private Properties generateRemoteDataFeedCachePropertiesFromMetadata(FeedUrl dat
         );
 
         final Properties properties = new Properties();
-        try {
-            String content = Downloader.getInstance().fetchContent(metaFeedUrl.toFormattedUrl("modified"), UTF_8);
-            final Properties props = new Properties();
-            props.load(new StringReader(content));
-            ZonedDateTime lmd = DatabaseProperties.getIsoTimestamp(props, "lastModifiedDate");
-            DatabaseProperties.setTimestamp(properties, "lastModifiedDate.modified", lmd);
-            DatabaseProperties.setTimestamp(properties, "lastModifiedDate", lmd);
-            final int startYear = settings.getInt(Settings.KEYS.NVD_API_DATAFEED_START_YEAR, 2002);
-            final ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC"));
-            final int endYear = now.withZoneSameInstant(ZoneId.of("UTC+14:00")).getYear();
-            for (int y = startYear; y <= endYear; y++) {
-                content = Downloader.getInstance().fetchContent(metaFeedUrl.toFormattedUrl(y), UTF_8);
-                props.clear();
-                props.load(new StringReader(content));
-                lmd = DatabaseProperties.getIsoTimestamp(props, "lastModifiedDate");
-                DatabaseProperties.setTimestamp(properties, "lastModifiedDate." + y, lmd);
-            }
-            return properties;
-        } catch (URISyntaxException | TooManyRequestsException | ResourceNotFoundException | IOException ex) {
-            throw new UpdateException("Unable to download the data feed META files", ex);
-        }
+        ZonedDateTime lmd = metaFeedUrl.getLastModifiedFor("modified");
+        DatabaseProperties.setTimestamp(properties, "lastModifiedDate.modified", lmd);
+        DatabaseProperties.setTimestamp(properties, "lastModifiedDate", lmd);
+
+        metaFeedUrl.getLastModifiedDatePropertiesByYear(this.settings, ZonedDateTime.now(ZoneOffset.UTC))
+                .forEach((k, v) -> DatabaseProperties.setTimestamp(properties, k, v));
+        return properties;
     }
 
     protected static class FeedUrl {
@@ -649,6 +639,15 @@ protected static class FeedUrl {
          */
         static final String DEFAULT_FILE_PATTERN = DEFAULT_FILE_PATTERN_PREFIX + DEFAULT_FILE_PATTERN_SUFFIX;
 
+        /**
+         * The timezone where the new year starts first.
+         */
+        static final ZoneId ZONE_GLOBAL_EARLIEST = ZoneId.of("UTC+14:00");
+        /**
+         * The timezone where the new year starts last.
+         */
+        static final ZoneId ZONE_GLOBAL_LATEST = ZoneId.of("UTC-12:00");
+
         /**
          * The base URL to download resources from.
          */
@@ -680,10 +679,6 @@ public FeedUrl withPattern(Function<Optional<String>, String> patternTransformer
             return new URI(toFormattedUrlString(formatArg)).toURL();
         }
 
-        @NotNull URL toFormattedUrl(int formatArg) throws MalformedURLException, URISyntaxException {
-            return toFormattedUrl(String.valueOf(formatArg));
-        }
-
         @SuppressWarnings("SameParameterValue")
         @NotNull URL toSuffixedUrl(String suffix) throws MalformedURLException, URISyntaxException {
             return new URI(url + suffix).toURL();
@@ -692,7 +687,7 @@ public FeedUrl withPattern(Function<Optional<String>, String> patternTransformer
         /**
          * @param url A NVD data feed URL which may be just a base URL such as https://my-nvd-cache/nvd_cache or
          *            may include a formatted URL ending with .json.gz such as https://nvd.nist.gov/feeds/json/cve/2.0/nvdcve-2.0-{0}.json.gz
-         * @return A constructed URLData object
+         * @return A constructed FeedUrl object
          */
         @SuppressWarnings("JavadocLinkAsPlainText")
         protected static FeedUrl extractFromUrlOptionalPattern(String url) {
@@ -710,5 +705,61 @@ protected static FeedUrl extractFromUrlOptionalPattern(String url) {
             }
             return new FeedUrl(baseUrl, pattern);
         }
+
+        private static @NonNull Pair<Integer, Integer> toYearRange(Settings settings, ZonedDateTime now) {
+            // for establishing the current year use the timezone where the new year starts first
+            // as from that moment on CNAs might start assigning CVEs with the new year depending
+            // on the CNA's timezone
+            final int startYear = settings.getInt(Settings.KEYS.NVD_API_DATAFEED_START_YEAR, 2002);
+            final int endYear = now.withZoneSameInstant(ZONE_GLOBAL_EARLIEST).getYear();
+            return new Pair<>(startYear, endYear);
+        }
+
+        private @Nullable ZonedDateTime getLastModifiedFor(String fileVersion) throws UpdateException {
+            try {
+                String content = Downloader.getInstance().fetchContent(toFormattedUrl(fileVersion), UTF_8);
+                Properties props = new Properties();
+                props.load(new StringReader(content));
+                return DatabaseProperties.getIsoTimestamp(props, "lastModifiedDate");
+            } catch (URISyntaxException | TooManyRequestsException | ResourceNotFoundException | IOException ex) {
+                throw new UpdateException("Unable to download the data feed .meta file for " + fileVersion, ex);
+            }
+        }
+
+        Map<String, ZonedDateTime> getLastModifiedDatePropertiesByYear(Settings settings, ZonedDateTime now) throws UpdateException {
+            Pair<Integer, Integer> yearRange = toYearRange(settings, now);
+            int startYear = yearRange.getLeft();
+            int endYear = yearRange.getRight();
+
+            Map<String, ZonedDateTime> lastModifiedDateProperties = new LinkedHashMap<>();
+            for (int y = startYear; y <= endYear; y++) {
+                try {
+                    lastModifiedDateProperties.put("lastModifiedDate." + y, getLastModifiedFor(String.valueOf(y)));
+                } catch (UpdateException e) {
+                    if (isMandatoryFeedYear(now, y)) {
+                        throw e;
+                    }
+                    LOGGER.debug("Ignoring data feed metadata retrieval failure for {}, it is still January 1st in some TZ; so feed files may not yet be generated. Error was {}", y, e.toString());
+                }
+            }
+            return lastModifiedDateProperties;
+        }
+
+        /**
+         * @param now        The current time in any timezone
+         * @param targetYear Target year's feed data to retrieve
+         * @return Whether or not the targetYear is considered a mandatory feed file to retrieve given the target year and current time.
+         */
+        static boolean isMandatoryFeedYear(ZonedDateTime now, int targetYear) {
+            return isNotTargetYearInAnyTZ(now, targetYear) || isAfterJanuary1InEveryTZ(now, targetYear);
+        }
+
+        private static boolean isNotTargetYearInAnyTZ(ZonedDateTime now, int targetYear) {
+            return targetYear != now.withZoneSameInstant(ZONE_GLOBAL_EARLIEST).getYear();
+        }
+
+        private static boolean isAfterJanuary1InEveryTZ(ZonedDateTime now, int targetYear) {
+            return now.isAfter(LocalDate.of(targetYear, 1, 2).atStartOfDay().atZone(ZONE_GLOBAL_LATEST));
+        }
     }
 }
diff --git a/core/src/test/java/org/owasp/dependencycheck/data/update/NvdApiDataSourceTest.java b/core/src/test/java/org/owasp/dependencycheck/data/update/NvdApiDataSourceTest.java
@@ -17,25 +17,42 @@
  */
 package org.owasp.dependencycheck.data.update;
 
+import org.hamcrest.Matchers;
+import org.jspecify.annotations.NonNull;
 import org.junit.jupiter.api.Nested;
 import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+import org.owasp.dependencycheck.data.update.exception.UpdateException;
+import org.owasp.dependencycheck.utils.DownloadFailedException;
+import org.owasp.dependencycheck.utils.Downloader;
+import org.owasp.dependencycheck.utils.Settings;
 
 import java.net.URI;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.util.List;
+import java.util.Map;
 import java.util.NoSuchElementException;
 
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.everyItem;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.when;
 import static org.owasp.dependencycheck.data.update.NvdApiDataSource.FeedUrl.DEFAULT_FILE_PATTERN;
 import static org.owasp.dependencycheck.data.update.NvdApiDataSource.FeedUrl.extractFromUrlOptionalPattern;
+import static org.owasp.dependencycheck.data.update.NvdApiDataSource.FeedUrl.isMandatoryFeedYear;
 
-/**
- *
- * @author Jeremy Long
- */
 class NvdApiDataSourceTest {
 
     @Nested
-    class FeedUrl {
+    class FeedUrlParsing {
 
         @Test
         void shouldExtractUrlWithPattern() throws Exception {
@@ -47,12 +64,12 @@ void shouldExtractUrlWithPattern() throws Exception {
             assertEquals(URI.create(expectedUrl).toURL(), result.toFormattedUrl("2045"));
             assertEquals(URI.create("https://internal.server/nist/some-file.txt").toURL(), result.toSuffixedUrl("some-file.txt"));
 
-            assertEquals(expectedUrl, result.toFormattedUrlString(2045));
-            assertEquals(URI.create(expectedUrl).toURL(), result.toFormattedUrl(2045));
+            assertEquals(expectedUrl, result.toFormattedUrlString("2045"));
+            assertEquals(URI.create(expectedUrl).toURL(), result.toFormattedUrl("2045"));
         }
 
         @Test
-        void shouldAllowTransformingFilePattern() throws Exception {
+        void shouldAllowTransformingFilePattern() {
             NvdApiDataSource.FeedUrl result = extractFromUrlOptionalPattern("https://internal.server/nist/nvdcve-{0}.json.gz")
                     .withPattern(p -> p.orElseThrow().replace(".json.gz", ".something"));
             assertEquals("https://internal.server/nist/nvdcve-ok.something", result.toFormattedUrlString("ok"));
@@ -80,7 +97,7 @@ void shouldExtractUrlWithoutPattern() throws Exception {
         }
 
         @Test
-        void extractUrlWithoutPatternShouldAddTrailingSlashes() throws Exception {
+        void extractUrlWithoutPatternShouldAddTrailingSlashes() {
             String nvdDataFeedUrl = "https://internal.server/nist";
             String expectedUrl = "https://internal.server/nist/nvdcve-2045.json.gz";
 
@@ -90,4 +107,115 @@ void extractUrlWithoutPatternShouldAddTrailingSlashes() throws Exception {
             assertEquals(expectedUrl, result.toFormattedUrlString("2045"));
         }
     }
+
+    @Nested
+    class FeedUrlMandatoryYears {
+
+        @Test
+        void shouldConsiderYearsMandatoryWhenNotCurrentYearAtEarliestTZ() {
+            ZonedDateTime janFirst2004AtEarliest = ZonedDateTime.of(2004, 1, 1, 0, 0, 0, 0, NvdApiDataSource.FeedUrl.ZONE_GLOBAL_EARLIEST);
+            assertTrue(isMandatoryFeedYear(janFirst2004AtEarliest, 2002));
+            assertTrue(isMandatoryFeedYear(janFirst2004AtEarliest, 2003));
+            assertFalse(isMandatoryFeedYear(janFirst2004AtEarliest, 2004));
+        }
+
+        @Test
+        void shouldConsiderYearsMandatoryWhenNotCurrentYearAtLatestTZ() {
+            ZonedDateTime janFirst2004AtLatest = ZonedDateTime.of(2004, 1, 1, 0, 0, 0, 0, NvdApiDataSource.FeedUrl.ZONE_GLOBAL_LATEST);
+            assertTrue(isMandatoryFeedYear(janFirst2004AtLatest, 2002));
+            assertTrue(isMandatoryFeedYear(janFirst2004AtLatest, 2003));
+            assertFalse(isMandatoryFeedYear(janFirst2004AtLatest, 2004));
+        }
+
+        @Test
+        void shouldConsiderYearsMandatoryWhenNoLongerJan1Anywhere() {
+            // It's still Jan 1 somewhere...
+            ZonedDateTime janSecond2004AtEarliest = ZonedDateTime.of(2004, 1, 2, 0, 0, 0, 0, NvdApiDataSource.FeedUrl.ZONE_GLOBAL_EARLIEST);
+            assertFalse(isMandatoryFeedYear(janSecond2004AtEarliest, 2004));
+
+            // Until it's no longer Jan 1 anywhere
+            ZonedDateTime janSecond2004AtLatest = ZonedDateTime.of(2004, 1, 2, 0, 0, 0, 1, NvdApiDataSource.FeedUrl.ZONE_GLOBAL_LATEST);
+            assertTrue(isMandatoryFeedYear(janSecond2004AtLatest, 2004));
+        }
+    }
+
+    @Nested
+    class FeedUrlMetadataRetrieval {
+
+        @Test
+        void shouldRetrieveMetadataByYear() throws Exception {
+            try (MockedStatic<Downloader> downloaderClass = mockStatic(Downloader.class)) {
+                Downloader downloader = mock(Downloader.class);
+                when(downloader.fetchContent(any(), any())).thenReturn("lastModifiedDate=2013-01-01T12:00:00Z");
+                downloaderClass.when(Downloader::getInstance).thenReturn(downloader);
+
+                assertThat(retrieveUntil(ZonedDateTime.of(2003, 12, 1, 0, 0, 0, 0, ZoneOffset.UTC)).keySet(),
+                        contains("lastModifiedDate.2002", "lastModifiedDate.2003"));
+            }
+        }
+
+        @Test
+        void shouldRetrieveMetadataForNextYearOnJan1AtEarliestTZ() throws Exception {
+            try (MockedStatic<Downloader> downloaderClass = mockStatic(Downloader.class)) {
+                Downloader downloader = mock(Downloader.class);
+                when(downloader.fetchContent(any(), any())).thenReturn("lastModifiedDate=2013-01-01T12:00:00Z");
+                downloaderClass.when(Downloader::getInstance).thenReturn(downloader);
+
+                ZonedDateTime jan1Earliest = ZonedDateTime.of(2004, 1, 1, 0, 0, 0, 0, NvdApiDataSource.FeedUrl.ZONE_GLOBAL_EARLIEST);
+                assertThat(retrieveUntil(jan1Earliest.minusSeconds(1)).keySet(),
+                        contains("lastModifiedDate.2002", "lastModifiedDate.2003"));
+
+                assertThat(retrieveUntil(jan1Earliest).keySet(),
+                        contains("lastModifiedDate.2002", "lastModifiedDate.2003", "lastModifiedDate.2004"));
+
+                assertThat(retrieveUntil(ZonedDateTime.of(2004, 1, 1, 0, 0, 0, 0, NvdApiDataSource.FeedUrl.ZONE_GLOBAL_LATEST)).keySet(),
+                        contains("lastModifiedDate.2002", "lastModifiedDate.2003", "lastModifiedDate.2004"));
+            }
+        }
+
+        @Test
+        void shouldNormallyRethrowDownloadErrorsEvenIfJan1OnEndYear() throws Exception {
+            try (MockedStatic<Downloader> downloaderClass = mockStatic(Downloader.class)) {
+                Downloader downloader = mock(Downloader.class);
+                when(downloader.fetchContent(any(), any())).thenThrow(new DownloadFailedException("failed to download"));
+                downloaderClass.when(Downloader::getInstance).thenReturn(downloader);
+
+                assertThrows(UpdateException.class, () -> retrieveUntil(ZonedDateTime.of(2003, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC)));
+            }
+        }
+
+        @Test
+        void shouldIgnoreDownloadFailureForFinalYearIfStillJan1() throws Exception {
+            List<ZonedDateTime> untilDates = List.of(
+                    ZonedDateTime.of(2004, 1, 1, 0, 0, 0, 0, NvdApiDataSource.FeedUrl.ZONE_GLOBAL_EARLIEST),
+                    ZonedDateTime.of(2004, 1, 2, 0, 0, 0, 0, NvdApiDataSource.FeedUrl.ZONE_GLOBAL_LATEST)
+                            .minusSeconds(1)
+            );
+
+            for (ZonedDateTime until : untilDates) {
+                try (MockedStatic<Downloader> downloaderClass = mockStatic(Downloader.class)) {
+                    Downloader downloader = mock(Downloader.class);
+                    when(downloader.fetchContent(any(), any()))
+                            .thenReturn("lastModifiedDate=2013-01-01T12:00:00Z")
+                            .thenReturn("lastModifiedDate=2013-01-01T12:00:00Z")
+                            .thenThrow(new DownloadFailedException("failed to download 3rd file"));
+
+                    downloaderClass.when(Downloader::getInstance).thenReturn(downloader);
+
+                    assertThat(retrieveUntil(until).keySet(),
+                            contains("lastModifiedDate.2002", "lastModifiedDate.2003"));
+                }
+            }
+        }
+
+        private @NonNull Map<String, ZonedDateTime> retrieveUntil(ZonedDateTime dec12th) throws UpdateException {
+            Map<String, ZonedDateTime> lastModifieds;
+            NvdApiDataSource.FeedUrl feedUrl = extractFromUrlOptionalPattern("https://internal.server/nist/nvdcve-{0}.json.gz");
+
+            lastModifieds = feedUrl.getLastModifiedDatePropertiesByYear(new Settings(), dec12th);
+
+            assertThat(lastModifieds.values(), everyItem(Matchers.equalTo(ZonedDateTime.of(2013, 1, 1, 12, 0, 0, 0, ZoneOffset.UTC))));
+            return lastModifieds;
+        }
+    }
 }
