Skip to content

Commit 1a9cfff

Browse files
mecheromechero
committed
Fixed bug when more than 10K issues, introduced a small cache.
1 parent 33744ae commit 1a9cfff

File tree

7 files changed

+242
-44
lines changed

7 files changed

+242
-44
lines changed

‎pom.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,12 @@
5454
<artifactId>commons-lang3</artifactId>
5555
<version>3.8.1</version>
5656
</dependency>
57+
<dependency>
58+
<groupId>com.google.guava</groupId>
59+
<artifactId>guava</artifactId>
60+
<version>29.0-jre</version>
61+
</dependency>
62+
5763
<!-- quboo -->
5864
<dependency>
5965
<groupId>com.fasterxml.jackson.core</groupId>

‎redeploy.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#!/usr/bin/env bash
22
mvn package
3-
cp -v ./target/*.jar /usr/local/Cellar/sonarqube/8.2.0.32929/libexec/extensions/plugins
3+
rm -rf /usr/local/Cellar/sonarqube/8.2.0.32929/libexec/extensions/plugins/quboo*.*
4+
cp -v ./target/*-SNAPSHOT.jar /usr/local/Cellar/sonarqube/8.2.0.32929/libexec/extensions/plugins
45
sonar restart

‎src/main/java/io/tpd/quboo/sonarplugin/dtos/IssuesWrapper.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,11 @@
22

33
import io.tpd.quboo.sonarplugin.pojos.Issue;
44
import io.tpd.quboo.sonarplugin.pojos.Issues;
5+
import io.tpd.quboo.sonarplugin.util.QubooCache;
56

67
import java.util.ArrayList;
78
import java.util.List;
9+
import java.util.stream.Collectors;
810

911
import static io.tpd.quboo.sonarplugin.QubooPlugin.QUBOO_API_VERSION;
1012

@@ -20,7 +22,7 @@ public IssuesWrapper() {
2022

2123
public void filterAndAddIssues(final Issues issues, final String sonarVersion) {
2224
this.issues.addAll(
23-
issues.getIssues()
25+
issues.getIssues().stream().filter(issue -> !QubooCache.INSTANCE.inCache(issue)).collect(Collectors.toList())
2426
);
2527
this.sonarVersion = sonarVersion;
2628
}

‎src/main/java/io/tpd/quboo/sonarplugin/dtos/UsersWrapper.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,11 @@
22

33
import io.tpd.quboo.sonarplugin.pojos.User;
44
import io.tpd.quboo.sonarplugin.pojos.Users;
5+
import io.tpd.quboo.sonarplugin.util.QubooCache;
56

67
import java.util.ArrayList;
78
import java.util.List;
9+
import java.util.stream.Collectors;
810

911
import static io.tpd.quboo.sonarplugin.QubooPlugin.QUBOO_API_VERSION;
1012

@@ -20,7 +22,8 @@ public UsersWrapper() {
2022

2123
public void filterAndAddUsers(final Users users, final String sonarVersion) {
2224
this.users.addAll(
23-
users.getUsers()
25+
users.getUsers().stream().filter(user -> !QubooCache.INSTANCE.inCache(user))
26+
.collect(Collectors.toList())
2427
);
2528
this.sonarVersion = sonarVersion;
2629
}

‎src/main/java/io/tpd/quboo/sonarplugin/hooks/QubooConnector.java‎

Lines changed: 78 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import io.tpd.quboo.sonarplugin.pojos.Paging;
1010
import io.tpd.quboo.sonarplugin.pojos.Users;
1111
import io.tpd.quboo.sonarplugin.settings.QubooProperties;
12+
import io.tpd.quboo.sonarplugin.util.QubooCache;
1213
import okhttp3.*;
1314
import org.sonar.api.ce.posttask.PostProjectAnalysisTask;
1415
import org.sonar.api.platform.Server;
@@ -68,50 +69,78 @@ public void finished(ProjectAnalysis analysis) {
6869
}
6970

7071
private void sendIssuesToQuboo(final IssuesWrapper allIssues, final String qubooKey, final String qubooSecret) throws Exception {
71-
final Request request = new Request.Builder()
72-
.url(QubooPlugin.QUBOO_SERVER + "/updater/issues")
73-
.header(QubooPlugin.QUBOO_HEADER_ACCESS_KEY, qubooKey)
74-
.header(QubooPlugin.QUBOO_HEADER_SECRET_KEY, qubooSecret)
75-
.post(RequestBody.create(MediaType.get("application/json"), mapper.writeValueAsString(allIssues)))
76-
.build();
77-
final Response response = http.newCall(request).execute();
78-
final String body = response.body().string();
79-
log.info("Response " + response.code() + " | " + body);
72+
if (!allIssues.getIssues().isEmpty()) {
73+
final Request request = new Request.Builder()
74+
.url(QubooPlugin.QUBOO_SERVER + "/updater/issues")
75+
.header(QubooPlugin.QUBOO_HEADER_ACCESS_KEY, qubooKey)
76+
.header(QubooPlugin.QUBOO_HEADER_SECRET_KEY, qubooSecret)
77+
.post(RequestBody.create(MediaType.get("application/json"), mapper.writeValueAsString(allIssues)))
78+
.build();
79+
log.info("Sending {} issues to Quboo, url: {}", allIssues.getIssues().size(), request.url().toString());
80+
final Response response = http.newCall(request).execute();
81+
final String body = response.body().string();
82+
log.info("Response " + response.code() + " | " + body);
83+
if (response.isSuccessful()) {
84+
QubooCache.INSTANCE.toCache(allIssues);
85+
}
86+
} else {
87+
log.info("No new issues to send to Quboo. Skipping...");
88+
}
8089
}
8190

82-
private IssuesWrapper getIssues(final String token) throws Exception {
91+
private IssuesWrapper getIssues(final String token) {
8392
int pageNumber = 1;
8493
boolean moreData = true;
8594
IssuesWrapper wrapper = new IssuesWrapper();
8695
while (moreData) {
87-
final Request.Builder request = new Request.Builder()
88-
.url(server.getPublicRootUrl() + "/api/issues/search?assigned=true&ps=200&p=" + pageNumber).get();
89-
addAuthorizationIfNeeded(request, token);
90-
final Request r = request.build();
91-
log.info("Quboo plugin getting issues from {}", r.url().toString());
92-
final Response response = http.newCall(r).execute();
93-
final String body = response.body().string();
94-
final Issues issues = mapper.readValue(body, Issues.class);
95-
log.info("Quboo plugin got {} issues", issues.getIssues().size());
96-
wrapper.filterAndAddIssues(issues, server.getVersion());
97-
moreData = moreData(issues.getPaging(), issues.getIssues().size());
98-
pageNumber++;
96+
try {
97+
final Request.Builder request = new Request.Builder()
98+
.url(server.getPublicRootUrl() + "/api/issues/search?assigned=true&ps=200&p=" + pageNumber).get();
99+
addAuthorizationIfNeeded(request, token);
100+
final Request r = request.build();
101+
log.info("Quboo plugin getting issues from {}", r.url().toString());
102+
final Response response = http.newCall(r).execute();
103+
if (response.isSuccessful()) {
104+
final String body = response.body().string();
105+
final Issues issues = mapper.readValue(body, Issues.class);
106+
log.info("Quboo plugin got {} issues", issues.getIssues().size());
107+
wrapper.filterAndAddIssues(issues, server.getVersion());
108+
moreData = moreData(issues.getPaging(), issues.getIssues().size());
109+
pageNumber++;
110+
if (pageNumber > 50) { // there is a hard limit in Sonar API that doesn't allow querying more than 10K results
111+
log.info("Reached max number of issues that can be fetched. Skipping remaining ones...");
112+
break;
113+
}
114+
} else {
115+
log.error("Aborting issues fetch. Got an error from server: {}. Request: {}", response.message(), r.url().toString());
116+
break;
117+
}
118+
} catch (final Exception e) {
119+
log.error("Quboo could not fetch issues from the server: " + e.getMessage());
120+
break;
121+
}
99122
}
100-
log.info("Sending " + wrapper.getIssues().size() + " issues to Quboo");
101123
return wrapper;
102124
}
103125

104126
private void sendUsersToQuboo(final UsersWrapper allUsers, final String qubooKey, final String qubooSecret) throws Exception {
105-
final Request request = new Request.Builder()
106-
.url(QubooPlugin.QUBOO_SERVER + "/updater/users")
107-
.header(QubooPlugin.QUBOO_HEADER_ACCESS_KEY, qubooKey)
108-
.header(QubooPlugin.QUBOO_HEADER_SECRET_KEY, qubooSecret)
109-
.post(RequestBody.create(MediaType.get("application/json"), mapper.writeValueAsString(allUsers)))
110-
.build();
111-
log.info("Sending users to Quboo, url: {}", request.url().toString());
112-
final Response response = http.newCall(request).execute();
113-
final String body = response.body().string();
114-
log.info("Response " + response.code() + " | " + body);
127+
if (!allUsers.getUsers().isEmpty()) {
128+
final Request request = new Request.Builder()
129+
.url(QubooPlugin.QUBOO_SERVER + "/updater/users")
130+
.header(QubooPlugin.QUBOO_HEADER_ACCESS_KEY, qubooKey)
131+
.header(QubooPlugin.QUBOO_HEADER_SECRET_KEY, qubooSecret)
132+
.post(RequestBody.create(MediaType.get("application/json"), mapper.writeValueAsString(allUsers)))
133+
.build();
134+
log.info("Sending {} users to Quboo, url: {}", allUsers.getUsers().size(), request.url().toString());
135+
final Response response = http.newCall(request).execute();
136+
final String body = response.body().string();
137+
log.info("Response " + response.code() + " | " + body);
138+
if (response.isSuccessful()) {
139+
QubooCache.INSTANCE.toCache(allUsers);
140+
}
141+
} else {
142+
log.info("No new users to send to Quboo. Skipping...");
143+
}
115144
}
116145

117146
private UsersWrapper getUsers(final String token) {
@@ -127,18 +156,26 @@ private UsersWrapper getUsers(final String token) {
127156
final Request r = request.build();
128157
log.info("Quboo plugin getting users from {}", r.url().toString());
129158
final Response response = http.newCall(r).execute();
130-
final String body = response.body().string();
131-
final Users users = mapper.readValue(body, Users.class);
132-
log.info("Quboo plugin got {} users", users.getUsers().size());
133-
wrapper.filterAndAddUsers(users, server.getVersion());
134-
moreData = moreData(users.getPaging(), users.getUsers().size());
135-
pageNumber++;
159+
if (response.isSuccessful()) {
160+
final String body = response.body().string();
161+
final Users users = mapper.readValue(body, Users.class);
162+
log.info("Quboo plugin got {} users", users.getUsers().size());
163+
wrapper.filterAndAddUsers(users, server.getVersion());
164+
moreData = moreData(users.getPaging(), users.getUsers().size());
165+
pageNumber++;
166+
if (pageNumber > 50) { // there is a hard limit in Sonar API that doesn't allow querying more than 10K results
167+
log.info("Reached max number of users that can be fetched. Skipping remaining ones...");
168+
break;
169+
}
170+
} else {
171+
log.error("Aborting users fetch. Got an error from server: {}. Request: {}", response.message(), r.url().toString());
172+
break;
173+
}
136174
} catch (final Exception e) {
137-
log.error("Quboo could not fetch data from the server: " + e.getMessage());
175+
log.error("Quboo could not fetch users from the server: " + e.getMessage());
138176
break;
139177
}
140178
}
141-
log.info("Sending " + wrapper.getUsers().size() + " users to Quboo");
142179
return wrapper;
143180
}
144181

‎src/main/java/io/tpd/quboo/sonarplugin/util/QubooCache.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
package io.tpd.quboo.sonarplugin.util;
2+
3+
import com.google.common.cache.Cache;
4+
import com.google.common.cache.CacheBuilder;
5+
import io.tpd.quboo.sonarplugin.dtos.IssuesWrapper;
6+
import io.tpd.quboo.sonarplugin.dtos.UsersWrapper;
7+
import io.tpd.quboo.sonarplugin.pojos.Issue;
8+
import io.tpd.quboo.sonarplugin.pojos.User;
9+
10+
import java.time.Duration;
11+
12+
/**
13+
* This cache avoids sending the same users and issues to the server over and over again.
14+
*/
15+
public enum QubooCache {
16+
17+
INSTANCE;
18+
19+
private Cache<String, String> usersCache;
20+
private Cache<String, String> issuesCache;
21+
22+
QubooCache() {
23+
usersCache = CacheBuilder.newBuilder().maximumSize(5000).expireAfterWrite(Duration.ofDays(2)).build();
24+
issuesCache = CacheBuilder.newBuilder().maximumSize(11000).expireAfterWrite(Duration.ofDays(2)).build();
25+
}
26+
27+
public void toCache(final UsersWrapper usersWrapper) {
28+
usersWrapper.getUsers().forEach(user -> usersCache.put(user.getLogin(), user.getName()));
29+
}
30+
31+
public void toCache(final IssuesWrapper issuesWrapper) {
32+
issuesWrapper.getIssues().forEach(issue -> issuesCache.put(issueHash(issue), issue.getKey()));
33+
}
34+
35+
public boolean inCache(final User user) {
36+
return usersCache.getIfPresent(user.getLogin()) != null;
37+
}
38+
39+
private String issueHash(final Issue issue) {
40+
return issue.getKey() + issue.getAssignee() + issue.getStatus() + issue.getResolution() + issue.getAuthor();
41+
}
42+
43+
public boolean inCache(final Issue issue) {
44+
return issuesCache.getIfPresent(issueHash(issue)) != null;
45+
}
46+
47+
public void clear() {
48+
issuesCache.invalidateAll();
49+
usersCache.invalidateAll();
50+
}
51+
}

0 commit comments

Comments
 (0)