Migrate NVD mirroring from hyades mirror-service

* Adds a new `VulnDataSource` extension to mirror contents of the NVD.
* Uses new JSON 2.0 feeds instead of NVD REST API.
* Modifies `NistMirrorTask` to no longer emit Kafka events, but instead invoke the new NVD `VulnDataSource`.

Closes DependencyTrack/hyades#1856

Signed-off-by: nscuro <[email protected]>

Author: nscuro

alpine/pom.xml

@@ -86,7 +86,6 @@
         <lib.javassist.version>3.30.2-GA</lib.javassist.version>
         <lib.jaxb-runtime.version>4.0.5</lib.jaxb-runtime.version>
         <lib.jdo.api.version>3.2.1</lib.jdo.api.version>
-        <lib.json-unit.version>4.1.1</lib.json-unit.version>
         <lib.jsonwebtoken.version>0.13.0</lib.jsonwebtoken.version>
         <lib.jsr305.version>3.0.2</lib.jsr305.version>
         <lib.logback.version>1.5.18</lib.logback.version>
@@ -268,12 +267,6 @@
                 <version>${lib.mockito.version}</version>
                 <scope>test</scope>
             </dependency>
-            <dependency>
-                <groupId>net.javacrumbs.json-unit</groupId>
-                <artifactId>json-unit-assertj</artifactId>
-                <version>${lib.json-unit.version}</version>
-                <scope>test</scope>
-            </dependency>
             <dependency>
                 <groupId>com.github.tomakehurst</groupId>
                 <artifactId>wiremock-jre8-standalone</artifactId>

api/src/main/openapi/components/schemas/extension-config-type.yaml

@@ -24,10 +24,12 @@ description: |-
   * `INTEGER` represents an integer, e.g. `666`.
   * `PATH` represents a file path, e.g. `/foo/bar.baz`.
   * `STRING` represents a simple string, e.g. `foo bar baz`.
+  * `URL` represents a uniform resource locator, e.g. `https://example.com`.
 enum:
 - BOOLEAN
 - DURATION
 - INSTANT
 - INTEGER
 - PATH
-- STRING
+- STRING
+- URL

api/src/main/openapi/components/schemas/update-extension-configs-request-item.yaml

@@ -29,5 +29,4 @@ properties:
       to provide the clear text secret again.
     type: string
 required:
-- name
-- value
+- name

apiserver/pom.xml

@@ -21,7 +21,6 @@
         <lib.awaitility.version>4.3.0</lib.awaitility.version>
         <lib.cloud-sql-postgres-socket-factory.version>1.25.3</lib.cloud-sql-postgres-socket-factory.version>
         <lib.commons-compress.version>1.28.0</lib.commons-compress.version>
-        <lib.cpe-parser.version>3.0.0</lib.cpe-parser.version>
         <lib.cvss-calculator.version>1.4.3</lib.cvss-calculator.version>
         <lib.owasp-rr-calculator.version>1.0.1</lib.owasp-rr-calculator.version>
         <lib.cyclonedx-java.version>10.2.1</lib.cyclonedx-java.version>
@@ -40,7 +39,6 @@
         <lib.swagger.version>2.2.37</lib.swagger.version>
         <lib.swagger-parser.version>2.1.34</lib.swagger-parser.version>
         <lib.system-rules.version>1.19.0</lib.system-rules.version>
-        <lib.versatile.version>0.7.0</lib.versatile.version>
         <lib.woodstox.version>7.1.1</lib.woodstox.version>
         <lib.junit-params.version>1.1.1</lib.junit-params.version>
         <lib.log4j-over-slf4j.version>2.0.17</lib.log4j-over-slf4j.version>
@@ -109,6 +107,12 @@
             <version>${project.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.dependencytrack</groupId>
+            <artifactId>vuln-data-source-nvd</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
         <!-- Alpine -->
         <dependency>
             <groupId>org.dependencytrack</groupId>
@@ -199,7 +203,6 @@
         <dependency>
             <groupId>us.springett</groupId>
             <artifactId>cpe-parser</artifactId>
-            <version>${lib.cpe-parser.version}</version>
         </dependency>
         <!-- CycloneDX -->
         <dependency>
@@ -330,7 +333,6 @@
         <dependency>
             <groupId>io.github.nscuro</groupId>
             <artifactId>versatile</artifactId>
-            <version>${lib.versatile.version}</version>
         </dependency>
         <dependency>
             <groupId>org.apache.maven</groupId>

apiserver/src/main/java/org/dependencytrack/event/kafka/KafkaEventConverter.java

@@ -25,7 +25,6 @@
 import org.dependencytrack.event.ComponentVulnerabilityAnalysisEvent;
 import org.dependencytrack.event.EpssMirrorEvent;
 import org.dependencytrack.event.GitHubAdvisoryMirrorEvent;
-import org.dependencytrack.event.NistMirrorEvent;
 import org.dependencytrack.event.OsvMirrorEvent;
 import org.dependencytrack.event.kafka.KafkaTopics.Topic;
 import org.dependencytrack.model.Vulnerability;
@@ -71,7 +70,6 @@ private KafkaEventConverter() {
             case ComponentRepositoryMetaAnalysisEvent e -> convert(e);
             case ComponentVulnerabilityAnalysisEvent e -> convert(e);
             case GitHubAdvisoryMirrorEvent e -> convert(e);
-            case NistMirrorEvent e -> convert(e);
             case OsvMirrorEvent e -> convert(e);
             case EpssMirrorEvent e -> convert(e);
             default -> throw new IllegalArgumentException("Unable to convert event " + event);
@@ -153,11 +151,6 @@ static KafkaEvent<String, String> convert(final GitHubAdvisoryMirrorEvent ignore
         return new KafkaEvent<>(KafkaTopics.VULNERABILITY_MIRROR_COMMAND, key, null);
     }
 
-    static KafkaEvent<String, String> convert(final NistMirrorEvent ignored) {
-        final String key = Vulnerability.Source.NVD.name();
-        return new KafkaEvent<>(KafkaTopics.VULNERABILITY_MIRROR_COMMAND, key, null);
-    }
-
     static KafkaEvent<String, String> convert(final OsvMirrorEvent event) {
         final String key = Vulnerability.Source.OSV.name();
         final String value = event.ecosystem();

apiserver/src/main/java/org/dependencytrack/model/ConfigPropertyConstants.java

@@ -67,9 +67,6 @@ public enum ConfigPropertyConstants {
     SCANNER_SNYK_CVSS_SOURCE("scanner", "snyk.cvss.source", "NVD", PropertyType.STRING, "Type of source to be prioritized for cvss calculation", ConfigPropertyAccessMode.READ_WRITE),
     SCANNER_SNYK_BASE_URL("scanner", "snyk.base.url", "https://api.snyk.io", PropertyType.URL, "Base Url pointing to the hostname and path for Snyk analysis", ConfigPropertyAccessMode.READ_WRITE),
     VULNERABILITY_POLICY_FILE_LAST_MODIFIED_HASH("vulnerability-policy", "vulnerability.policy.file.last.modified.hash", null,  PropertyType.STRING, "Hash value or etag of the last fetched bundle if any", ConfigPropertyAccessMode.READ_ONLY),
-    VULNERABILITY_SOURCE_NVD_ENABLED("vuln-source", "nvd.enabled", "true", PropertyType.BOOLEAN, "Flag to enable/disable National Vulnerability Database", ConfigPropertyAccessMode.READ_WRITE),
-    VULNERABILITY_SOURCE_NVD_API_URL("vuln-source", "nvd.api.url", "https://services.nvd.nist.gov/rest/json/cves/2.0", PropertyType.URL, "REST API URL for the NVD's CVE API 2.0", ConfigPropertyAccessMode.READ_WRITE),
-    VULNERABILITY_SOURCE_NVD_API_KEY("vuln-source", "nvd.api.key", null, PropertyType.ENCRYPTEDSTRING, "API key for the NVD REST API", ConfigPropertyAccessMode.READ_WRITE),
     VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ENABLED("vuln-source", "github.advisories.enabled", "false", PropertyType.BOOLEAN, "Flag to enable/disable GitHub Advisories", ConfigPropertyAccessMode.READ_WRITE),
     VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ALIAS_SYNC_ENABLED("vuln-source", "github.advisories.alias.sync.enabled", "true", PropertyType.BOOLEAN, "Flag to enable/disable alias synchronization for GitHub Advisories", ConfigPropertyAccessMode.READ_WRITE),
     VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ACCESS_TOKEN("vuln-source", "github.advisories.access.token", null, PropertyType.STRING, "The access token used for GitHub API authentication", ConfigPropertyAccessMode.READ_WRITE),

apiserver/src/main/java/org/dependencytrack/resources/v2/ExtensionsResource.java

@@ -157,6 +157,7 @@ public Response listExtensionConfigs(
                                     case ConfigType.Integer ignored -> ExtensionConfigType.INTEGER;
                                     case ConfigType.Path ignored -> ExtensionConfigType.PATH;
                                     case ConfigType.String ignored -> ExtensionConfigType.STRING;
+                                    case ConfigType.URL ignored -> ExtensionConfigType.URL;
                                 })
                                 .isRequired(configDef.isRequired())
                                 .isSecret(configDef.isSecret())

apiserver/src/main/java/org/dependencytrack/tasks/NistMirrorTask.java

@@ -18,40 +18,154 @@
  */
 package org.dependencytrack.tasks;
 
-import alpine.common.logging.Logger;
 import alpine.event.framework.Event;
 import alpine.event.framework.LoggableSubscriber;
-import alpine.model.ConfigProperty;
+import net.javacrumbs.shedlock.core.LockConfiguration;
+import net.javacrumbs.shedlock.core.LockExtender;
+import net.javacrumbs.shedlock.core.LockingTaskExecutor;
+import org.cyclonedx.proto.v1_6.Bom;
 import org.dependencytrack.event.NistMirrorEvent;
-import org.dependencytrack.event.kafka.KafkaEventDispatcher;
+import org.dependencytrack.model.Vulnerability;
+import org.dependencytrack.model.VulnerableSoftware;
+import org.dependencytrack.parser.dependencytrack.BovModelConverter;
 import org.dependencytrack.persistence.QueryManager;
+import org.dependencytrack.plugin.PluginManager;
+import org.dependencytrack.plugin.api.datasource.vuln.VulnDataSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_NVD_ENABLED;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
 
+import static org.datanucleus.PropertyNames.PROPERTY_PERSISTENCE_BY_REACHABILITY_AT_COMMIT;
+import static org.dependencytrack.util.LockProvider.executeWithLock;
+import static org.dependencytrack.util.LockProvider.isTaskLockToBeExtended;
+import static org.dependencytrack.util.TaskUtil.getLockConfigForTask;
+
+/**
+ * Task for mirroring vulnerability data from the national vulnerability database (NVD).
+ * <p>
+ * The logic in this task is not NVD-specific and should eventually be used for all
+ * vulnerability data sources.
+ */
 public class NistMirrorTask implements LoggableSubscriber {
 
-    private static final Logger LOGGER = Logger.getLogger(NistMirrorTask.class);
+    private static final Logger LOGGER = LoggerFactory.getLogger(NistMirrorTask.class);
+
+    private final PluginManager pluginManager;
+    private LockConfiguration lockConfig;
+    private Instant lockAcquiredAt;
+
+    NistMirrorTask(PluginManager pluginManager) {
+        this.pluginManager = pluginManager;
+    }
+
+    @SuppressWarnings("unused")
+    public NistMirrorTask() {
+        this(PluginManager.getInstance());
+    }
 
-    /**
-     * {@inheritDoc}
-     */
     public void inform(final Event e) {
-        if (e instanceof NistMirrorEvent) {
-            try (final QueryManager qm = new QueryManager()) {
-                final ConfigProperty enabled = qm.getConfigProperty(VULNERABILITY_SOURCE_NVD_ENABLED.getGroupName(), VULNERABILITY_SOURCE_NVD_ENABLED.getPropertyName());
-                final boolean isEnabled = enabled != null && Boolean.valueOf(enabled.getPropertyValue());
-                if (!isEnabled) {
-                    return;
+        if (!(e instanceof NistMirrorEvent)) {
+            return;
+        }
+
+        lockConfig = getLockConfigForTask(getClass());
+
+        try {
+            executeWithLock(
+                    this.lockConfig,
+                    (LockingTaskExecutor.Task) this::informLocked);
+        } catch (Throwable ex) {
+            LOGGER.error("Failed to acquire lock or execute task", ex);
+        }
+    }
+
+    private void informLocked() {
+        lockAcquiredAt = Instant.now();
+
+        try (final var dataSource = pluginManager.getExtension(VulnDataSource.class, "nvd")) {
+            if (dataSource == null) {
+                return; // Likely disabled.
+            }
+
+            final var bovBatch = new ArrayList<Bom>(25);
+            while (dataSource.hasNext()) {
+                if (Thread.currentThread().isInterrupted()) {
+                    LOGGER.warn("Interrupted before all BOVs could be consumed");
+                    break;
                 }
 
-                final long start = System.currentTimeMillis();
-                LOGGER.info("Starting NIST mirroring task");
-                new KafkaEventDispatcher().dispatchEvent(new NistMirrorEvent()).join();
-                final long end = System.currentTimeMillis();
-                LOGGER.info("NIST mirroring complete. Time spent (total): " + (end - start) + "ms");
-            } catch (Exception ex) {
-                LOGGER.error("An unexpected error occurred while triggering NIST mirroring", ex);
+                maybeExtendLock();
+
+                bovBatch.add(dataSource.next());
+                if (bovBatch.size() == 25) {
+                    processBatch(dataSource, bovBatch);
+                    bovBatch.clear();
+                }
+            }
+
+            if (!bovBatch.isEmpty()) {
+                maybeExtendLock();
+                processBatch(dataSource, bovBatch);
+                bovBatch.clear();
             }
         }
     }
+
+    private void processBatch(final VulnDataSource dataSource, final Collection<Bom> bovs) {
+        LOGGER.debug("Processing batch of {} BOVs", bovs.size());
+
+        final var vulns = new ArrayList<Vulnerability>(bovs.size());
+        final var vsListByVulnId = new HashMap<String, List<VulnerableSoftware>>(bovs.size());
+
+        for (final Bom bov : bovs) {
+            if (bov.getVulnerabilitiesCount() == 0) {
+                LOGGER.warn("BOV contains no vulnerabilities; Skipping");
+                continue;
+            }
+
+            if (bov.getVulnerabilitiesCount() > 1) {
+                LOGGER.warn("BOV contains more than one vulnerability; Skipping");
+                continue;
+            }
+
+            final Vulnerability vuln = BovModelConverter.convert(bov, bov.getVulnerabilities(0), true);
+            final List<VulnerableSoftware> vsList = BovModelConverter.extractVulnerableSoftware(bov);
+
+            vulns.add(vuln);
+            vsListByVulnId.put(vuln.getVulnId(), vsList);
+        }
+
+        try (final var qm = new QueryManager()) {
+            qm.getPersistenceManager().setProperty(PROPERTY_PERSISTENCE_BY_REACHABILITY_AT_COMMIT, "false");
+
+            qm.runInTransaction(() -> {
+                for (final Vulnerability vuln : vulns) {
+                    LOGGER.debug("Synchronizing vulnerability {}", vuln.getVulnId());
+                    final Vulnerability persistentVuln = qm.synchronizeVulnerability(vuln, false);
+                    final List<VulnerableSoftware> vsList = vsListByVulnId.get(persistentVuln.getVulnId());
+                    qm.synchronizeVulnerableSoftware(persistentVuln, vsList, Vulnerability.Source.NVD);
+                }
+            });
+        }
+
+        for (final Bom bov : bovs) {
+            dataSource.markProcessed(bov);
+        }
+    }
+
+    private void maybeExtendLock() {
+        final var lockAge = Duration.between(lockAcquiredAt, Instant.now());
+        if (isTaskLockToBeExtended(lockAge.toMillis(), getClass())) {
+            LOGGER.warn("Extending lock by {}", lockConfig.getLockAtMostFor());
+            LockExtender.extendActiveLock(lockConfig.getLockAtMostFor(), this.lockConfig.getLockAtLeastFor());
+            lockAcquiredAt = Instant.now();
+        }
+    }
+
 }

apiserver/src/main/java/org/dependencytrack/util/VulnerabilityUtil.java

@@ -26,6 +26,9 @@
 import org.dependencytrack.model.Vulnerability;
 import org.dependencytrack.model.VulnerabilityAlias;
 import org.dependencytrack.persistence.QueryManager;
+import org.dependencytrack.plugin.NoSuchExtensionException;
+import org.dependencytrack.plugin.PluginManager;
+import org.dependencytrack.plugin.api.datasource.vuln.VulnDataSource;
 
 import java.math.BigDecimal;
 import java.security.SecureRandom;
@@ -276,9 +279,15 @@ public static boolean canBeMirrored(final Vulnerability vulnerability) {
      */
     public static boolean isMirroringEnabled(final Vulnerability vulnerability) {
         final Vulnerability.Source source = Vulnerability.Source.valueOf(vulnerability.getSource());
+        if (Vulnerability.Source.NVD == source) {
+            try (final var dataSource = PluginManager.getInstance().getExtension(VulnDataSource.class, "nvd")) {
+                return dataSource != null;
+            } catch (NoSuchExtensionException e) {
+                return false;
+            }
+        }
 
         final ConfigPropertyConstants toggleConfigPropertyConstant = switch (source) {
-            case NVD -> ConfigPropertyConstants.VULNERABILITY_SOURCE_NVD_ENABLED;
             case GITHUB -> ConfigPropertyConstants.VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ENABLED;
             case OSV -> ConfigPropertyConstants.VULNERABILITY_SOURCE_GOOGLE_OSV_ENABLED;
             default -> null;

apiserver/src/main/resources/application.properties

@@ -1050,6 +1050,24 @@ task.osv.mirror.cron=0 3 * * *
 # @required
 task.nist.mirror.cron=0 4 * * *
 
+# Maximum duration in ISO 8601 format for which the NIST mirror task will hold a lock.
+# <br/><br/>
+# The duration should be long enough to cover the task's execution duration.
+#
+# @category: Task Scheduling
+# @type:     duration
+# @required
+task.nist.mirror.lock.max.duration=PT15M
+
+# Minimum duration in ISO 8601 format for which the NIST mirror task will hold a lock.
+# <br/><br/>
+# The duration should be long enough to cover eventual clock skew across API server instances.
+#
+# @category: Task Scheduling
+# @type:     duration
+# @required
+task.nist.mirror.lock.min.duration=PT1M
+
 # Cron expression of the EPSS mirroring task.
 #
 # @category: Task Scheduling