Skip to content

Commit af5b23e

Browse files
nscuronscuro
committed
Migrate GitHub mirroring from hyades mirror-service
* Adds a new `VulnDataSource` extension to mirror contents of the GitHub Advisories database. * Modifies `GitHubAdvisoryMirrorTask` to no longer emit Kafka events, but instead invoke the new GitHub `VulnDataSource`. Signed-off-by: nscuro <[email protected]>
1 parent 3eec1d0 commit af5b23e

File tree

32 files changed

+2527
-211
lines changed

32 files changed

+2527
-211
lines changed

‎alpine/pom.xml‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,6 @@
9999
<lib.swagger.version>2.2.37</lib.swagger.version>
100100
<!-- Unit test libraries -->
101101
<lib.junit-pioneer.version>2.3.0</lib.junit-pioneer.version>
102-
<lib.mockito.version>5.20.0</lib.mockito.version>
103102
<lib.wiremock.version>2.35.2</lib.wiremock.version>
104103
</properties>
105104

@@ -261,12 +260,6 @@
261260
</exclusion>
262261
</exclusions>
263262
</dependency>
264-
<dependency>
265-
<groupId>org.mockito</groupId>
266-
<artifactId>mockito-core</artifactId>
267-
<version>${lib.mockito.version}</version>
268-
<scope>test</scope>
269-
</dependency>
270263
<dependency>
271264
<groupId>com.github.tomakehurst</groupId>
272265
<artifactId>wiremock-jre8-standalone</artifactId>

‎apiserver/pom.xml‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
<lib.awaitility.version>4.3.0</lib.awaitility.version>
2222
<lib.cloud-sql-postgres-socket-factory.version>1.25.3</lib.cloud-sql-postgres-socket-factory.version>
2323
<lib.commons-compress.version>1.28.0</lib.commons-compress.version>
24-
<lib.cvss-calculator.version>1.4.3</lib.cvss-calculator.version>
2524
<lib.owasp-rr-calculator.version>1.0.1</lib.owasp-rr-calculator.version>
2625
<lib.cyclonedx-java.version>10.2.1</lib.cyclonedx-java.version>
2726
<lib.jaxb.runtime.version>4.0.5</lib.jaxb.runtime.version>
@@ -107,6 +106,11 @@
107106
<version>${project.version}</version>
108107
</dependency>
109108

109+
<dependency>
110+
<groupId>org.dependencytrack</groupId>
111+
<artifactId>vuln-data-source-github</artifactId>
112+
<version>${project.version}</version>
113+
</dependency>
110114
<dependency>
111115
<groupId>org.dependencytrack</groupId>
112116
<artifactId>vuln-data-source-nvd</artifactId>
@@ -167,7 +171,6 @@
167171
<dependency>
168172
<groupId>us.springett</groupId>
169173
<artifactId>cvss-calculator</artifactId>
170-
<version>${lib.cvss-calculator.version}</version>
171174
</dependency>
172175
<dependency>
173176
<groupId>org.jdbi</groupId>

‎apiserver/src/main/java/org/dependencytrack/event/kafka/KafkaEventConverter.java‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent;
2525
import org.dependencytrack.event.ComponentVulnerabilityAnalysisEvent;
2626
import org.dependencytrack.event.EpssMirrorEvent;
27-
import org.dependencytrack.event.GitHubAdvisoryMirrorEvent;
2827
import org.dependencytrack.event.OsvMirrorEvent;
2928
import org.dependencytrack.event.kafka.KafkaTopics.Topic;
3029
import org.dependencytrack.model.Vulnerability;
@@ -69,7 +68,6 @@ private KafkaEventConverter() {
6968
return switch (event) {
7069
case ComponentRepositoryMetaAnalysisEvent e -> convert(e);
7170
case ComponentVulnerabilityAnalysisEvent e -> convert(e);
72-
case GitHubAdvisoryMirrorEvent e -> convert(e);
7371
case OsvMirrorEvent e -> convert(e);
7472
case EpssMirrorEvent e -> convert(e);
7573
default -> throw new IllegalArgumentException("Unable to convert event " + event);
@@ -146,11 +144,6 @@ static KafkaEvent<String, AnalysisCommand> convert(final ComponentRepositoryMeta
146144
return new KafkaEvent<>(KafkaTopics.REPO_META_ANALYSIS_COMMAND, event.purlCoordinates(), analysisCommand, null);
147145
}
148146

149-
static KafkaEvent<String, String> convert(final GitHubAdvisoryMirrorEvent ignored) {
150-
final String key = Vulnerability.Source.GITHUB.name();
151-
return new KafkaEvent<>(KafkaTopics.VULNERABILITY_MIRROR_COMMAND, key, null);
152-
}
153-
154147
static KafkaEvent<String, String> convert(final OsvMirrorEvent event) {
155148
final String key = Vulnerability.Source.OSV.name();
156149
final String value = event.ecosystem();

‎apiserver/src/main/java/org/dependencytrack/plugin/PluginManager.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,19 @@ public <T extends ExtensionPoint, U extends ExtensionFactory<T>> U getFactory(fi
143143
return (U) factory;
144144
}
145145

146+
@SuppressWarnings("unchecked")
147+
public <T extends ExtensionPoint, U extends ExtensionFactory<T>> U getFactory(final Class<T> extensionPointClass, final String name) {
148+
final var extensionIdentity = new ExtensionIdentity(extensionPointClass, name);
149+
final ExtensionFactory<?> factory = factoryByExtensionIdentity.get(extensionIdentity);
150+
if (factory == null) {
151+
throw new NoSuchExtensionException(
152+
"No factory for extension named %s exists for the extension point %s".formatted(
153+
name, extensionPointClass.getName()));
154+
}
155+
156+
return (U) factory;
157+
}
158+
146159
@SuppressWarnings("unchecked")
147160
public <T extends ExtensionPoint, U extends ExtensionFactory<T>> SequencedCollection<U> getFactories(final Class<T> extensionPointClass) {
148161
final Set<String> extensionNames = extensionNamesByExtensionPointClass.get(extensionPointClass);

‎apiserver/src/main/java/org/dependencytrack/tasks/AbstractVulnDataSourceMirrorTask.java‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
/*
2+
* This file is part of Dependency-Track.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
* SPDX-License-Identifier: Apache-2.0
17+
* Copyright (c) OWASP Foundation. All Rights Reserved.
18+
*/
19+
package org.dependencytrack.tasks;
20+
21+
import alpine.event.framework.Event;
22+
import alpine.event.framework.Subscriber;
23+
import com.google.protobuf.util.Timestamps;
24+
import net.javacrumbs.shedlock.core.LockConfiguration;
25+
import net.javacrumbs.shedlock.core.LockExtender;
26+
import net.javacrumbs.shedlock.core.LockingTaskExecutor;
27+
import org.cyclonedx.proto.v1_6.Bom;
28+
import org.dependencytrack.model.Vulnerability;
29+
import org.dependencytrack.model.VulnerableSoftware;
30+
import org.dependencytrack.parser.dependencytrack.BovModelConverter;
31+
import org.dependencytrack.persistence.QueryManager;
32+
import org.dependencytrack.plugin.PluginManager;
33+
import org.dependencytrack.plugin.api.datasource.vuln.VulnDataSource;
34+
import org.slf4j.Logger;
35+
import org.slf4j.LoggerFactory;
36+
37+
import java.time.Duration;
38+
import java.time.Instant;
39+
import java.util.ArrayList;
40+
import java.util.Collection;
41+
import java.util.HashMap;
42+
import java.util.List;
43+
44+
import static org.datanucleus.PropertyNames.PROPERTY_PERSISTENCE_BY_REACHABILITY_AT_COMMIT;
45+
import static org.dependencytrack.util.LockProvider.executeWithLock;
46+
import static org.dependencytrack.util.LockProvider.isTaskLockToBeExtended;
47+
import static org.dependencytrack.util.TaskUtil.getLockConfigForTask;
48+
49+
/**
50+
* @since 5.7.0
51+
*/
52+
abstract class AbstractVulnDataSourceMirrorTask implements Subscriber {
53+
54+
private final PluginManager pluginManager;
55+
private final Class<? extends Event> eventClass;
56+
private final String vulnDataSourceExtensionName;
57+
private final Vulnerability.Source source;
58+
private final Logger logger;
59+
private LockConfiguration lockConfig;
60+
private Instant lockAcquiredAt;
61+
62+
AbstractVulnDataSourceMirrorTask(
63+
final PluginManager pluginManager,
64+
final Class<? extends Event> eventClass,
65+
final String vulnDataSourceExtensionName,
66+
final Vulnerability.Source source) {
67+
this.pluginManager = pluginManager;
68+
this.eventClass = eventClass;
69+
this.vulnDataSourceExtensionName = vulnDataSourceExtensionName;
70+
this.source = source;
71+
this.logger = LoggerFactory.getLogger(this.getClass());
72+
}
73+
74+
@Override
75+
public void inform(final Event e) {
76+
if (!eventClass.isAssignableFrom(e.getClass())) {
77+
return;
78+
}
79+
80+
lockConfig = getLockConfigForTask(getClass());
81+
82+
try {
83+
executeWithLock(
84+
this.lockConfig,
85+
(LockingTaskExecutor.Task) this::informLocked);
86+
} catch (Throwable ex) {
87+
logger.error("Failed to acquire lock or execute task", ex);
88+
}
89+
}
90+
91+
private void informLocked() {
92+
lockAcquiredAt = Instant.now();
93+
94+
try (final var dataSource = pluginManager.getExtension(VulnDataSource.class, vulnDataSourceExtensionName)) {
95+
if (dataSource == null) {
96+
return; // Likely disabled.
97+
}
98+
99+
final var bovBatch = new ArrayList<Bom>(25);
100+
while (dataSource.hasNext()) {
101+
if (Thread.currentThread().isInterrupted()) {
102+
logger.warn("Interrupted before all BOVs could be consumed");
103+
break;
104+
}
105+
106+
maybeExtendLock();
107+
108+
final Bom bov = dataSource.next();
109+
if (!bov.getVulnerabilities(0).hasRejected()) {
110+
bovBatch.add(bov);
111+
if (bovBatch.size() == 25) {
112+
processBatch(dataSource, bovBatch);
113+
bovBatch.clear();
114+
}
115+
} else {
116+
// TODO: Store rejection / withdrawal timestamp instead,
117+
// and let analyzers / users decide how to deal with them.
118+
// Ignoring withdrawn vulnerabilities is legacy behavior.
119+
logger.warn(
120+
"Skipping vulnerability {} rejected at {}",
121+
bov.getVulnerabilities(0).getId(),
122+
Timestamps.toString(bov.getVulnerabilities(0).getRejected()));
123+
}
124+
}
125+
126+
if (!bovBatch.isEmpty()) {
127+
maybeExtendLock();
128+
processBatch(dataSource, bovBatch);
129+
bovBatch.clear();
130+
}
131+
}
132+
}
133+
134+
private void processBatch(final VulnDataSource dataSource, final Collection<Bom> bovs) {
135+
logger.debug("Processing batch of {} BOVs", bovs.size());
136+
137+
final var vulns = new ArrayList<Vulnerability>(bovs.size());
138+
final var vsListByVulnId = new HashMap<String, List<VulnerableSoftware>>(bovs.size());
139+
140+
for (final Bom bov : bovs) {
141+
if (bov.getVulnerabilitiesCount() == 0) {
142+
logger.warn("BOV contains no vulnerabilities; Skipping");
143+
continue;
144+
}
145+
146+
if (bov.getVulnerabilitiesCount() > 1) {
147+
logger.warn("BOV contains more than one vulnerability; Skipping");
148+
continue;
149+
}
150+
151+
final Vulnerability vuln = BovModelConverter.convert(bov, bov.getVulnerabilities(0), true);
152+
final List<VulnerableSoftware> vsList = BovModelConverter.extractVulnerableSoftware(bov);
153+
154+
vulns.add(vuln);
155+
vsListByVulnId.put(vuln.getVulnId(), vsList);
156+
}
157+
158+
try (final var qm = new QueryManager()) {
159+
qm.getPersistenceManager().setProperty(PROPERTY_PERSISTENCE_BY_REACHABILITY_AT_COMMIT, "false");
160+
161+
qm.runInTransaction(() -> {
162+
for (final Vulnerability vuln : vulns) {
163+
logger.debug("Synchronizing vulnerability {}", vuln.getVulnId());
164+
final Vulnerability persistentVuln = qm.synchronizeVulnerability(vuln, false);
165+
final List<VulnerableSoftware> vsList = vsListByVulnId.get(persistentVuln.getVulnId());
166+
qm.synchronizeVulnerableSoftware(persistentVuln, vsList, this.source);
167+
}
168+
});
169+
}
170+
171+
for (final Bom bov : bovs) {
172+
dataSource.markProcessed(bov);
173+
}
174+
}
175+
176+
private void maybeExtendLock() {
177+
final var lockAge = Duration.between(lockAcquiredAt, Instant.now());
178+
if (isTaskLockToBeExtended(lockAge.toMillis(), getClass())) {
179+
logger.warn("Extending lock by {}", lockConfig.getLockAtMostFor());
180+
LockExtender.extendActiveLock(lockConfig.getLockAtMostFor(), this.lockConfig.getLockAtLeastFor());
181+
lockAcquiredAt = Instant.now();
182+
}
183+
}
184+
185+
}

‎apiserver/src/main/java/org/dependencytrack/tasks/GitHubAdvisoryMirrorTask.java‎

Lines changed: 11 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -18,45 +18,22 @@
1818
*/
1919
package org.dependencytrack.tasks;
2020

21-
import alpine.common.logging.Logger;
22-
import alpine.event.framework.Event;
23-
import alpine.event.framework.LoggableSubscriber;
24-
import alpine.model.ConfigProperty;
2521
import org.dependencytrack.event.GitHubAdvisoryMirrorEvent;
26-
import org.dependencytrack.event.kafka.KafkaEventDispatcher;
27-
import org.dependencytrack.persistence.QueryManager;
22+
import org.dependencytrack.model.Vulnerability;
23+
import org.dependencytrack.plugin.PluginManager;
2824

29-
import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ACCESS_TOKEN;
30-
import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ENABLED;
31-
32-
public class GitHubAdvisoryMirrorTask implements LoggableSubscriber {
25+
/**
26+
* Task for mirroring vulnerability data from the GitHub Advisory database.
27+
*/
28+
public class GitHubAdvisoryMirrorTask extends AbstractVulnDataSourceMirrorTask {
3329

34-
private static final Logger LOGGER = Logger.getLogger(GitHubAdvisoryMirrorTask.class);
35-
private final boolean isEnabled;
36-
private String accessToken;
30+
GitHubAdvisoryMirrorTask(final PluginManager pluginManager) {
31+
super(pluginManager, GitHubAdvisoryMirrorEvent.class, "github", Vulnerability.Source.GITHUB);
32+
}
3733

34+
@SuppressWarnings("unused")
3835
public GitHubAdvisoryMirrorTask() {
39-
try (final QueryManager qm = new QueryManager()) {
40-
final ConfigProperty enabled = qm.getConfigProperty(VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ENABLED.getGroupName(), VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ENABLED.getPropertyName());
41-
this.isEnabled = enabled != null && Boolean.valueOf(enabled.getPropertyValue());
42-
final ConfigProperty accessToken = qm.getConfigProperty(VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ACCESS_TOKEN.getGroupName(), VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ACCESS_TOKEN.getPropertyName());
43-
if (accessToken != null) {
44-
this.accessToken = accessToken.getPropertyValue();
45-
}
46-
}
36+
this(PluginManager.getInstance());
4737
}
4838

49-
/**
50-
* {@inheritDoc}
51-
*/
52-
public void inform(final Event e) {
53-
if (e instanceof GitHubAdvisoryMirrorEvent && this.isEnabled) {
54-
if (this.accessToken != null) {
55-
LOGGER.info("Starting GitHub Advisory mirroring task");
56-
new KafkaEventDispatcher().dispatchEvent(new GitHubAdvisoryMirrorEvent()).join();
57-
} else {
58-
LOGGER.warn("GitHub Advisory mirroring is enabled, but no personal access token is configured. Skipping.");
59-
}
60-
}
61-
}
6239
}

0 commit comments

Comments
 (0)