Skip to content

Commit cce4eb5

Browse files
vladakvladak
committed
make markIndexed() asynchronous
1 parent d630fdc commit cce4eb5

File tree

5 files changed

+135
-76
lines changed

5 files changed

+135
-76
lines changed

‎apiary.apib

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -363,6 +363,8 @@ This entry point is used by the Indexer once it finishes indexing given project.
363363

364364
### marks project as indexed [PUT]
365365

366+
This is asynchronous API endpoint.
367+
366368
+ Request (text/plain)
367369
+ Body
368370

‎opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java

Lines changed: 2 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@
7777
import org.opengrok.indexer.util.PathUtils;
7878
import org.opengrok.indexer.util.ResourceLock;
7979
import org.opengrok.indexer.util.Statistics;
80+
import org.opengrok.indexer.web.ApiUtils;
8081
import org.opengrok.indexer.web.Prefix;
8182
import org.opengrok.indexer.web.Util;
8283
import org.opengrok.indexer.web.messages.Message;
@@ -1457,53 +1458,6 @@ public String getConfigurationXML() {
14571458
return syncReadConfiguration(Configuration::getXMLRepresentationAsString);
14581459
}
14591460

1460-
/**
1461-
* Busy waits for API call to complete by repeatedly querying the status API endpoint passed
1462-
* in the {@code Location} header in the response parameter. The overall time is governed
1463-
* by the {@link #getApiTimeout()}, however each individual status check
1464-
* uses {@link #getConnectTimeout()} so in the worst case the total time can be
1465-
* {@code getApiTimeout() * getConnectTimeout()}.
1466-
* @param response response returned from the server upon asynchronous API request
1467-
* @return response from the status API call
1468-
* @throws InterruptedException on sleep interruption
1469-
* @throws IllegalArgumentException on invalid request (no {@code Location} header)
1470-
*/
1471-
private @NotNull Response waitForAsyncApi(@NotNull Response response)
1472-
throws InterruptedException, IllegalArgumentException {
1473-
1474-
String location = response.getHeaderString(HttpHeaders.LOCATION);
1475-
if (location == null) {
1476-
throw new IllegalArgumentException(String.format("no %s header in %s", HttpHeaders.LOCATION, response));
1477-
}
1478-
1479-
LOGGER.log(Level.FINER, "checking asynchronous API result on {0}", location);
1480-
for (int i = 0; i < getApiTimeout(); i++) {
1481-
response = ClientBuilder.newBuilder().
1482-
connectTimeout(RuntimeEnvironment.getInstance().getConnectTimeout(), TimeUnit.SECONDS).build().
1483-
target(location).request().get();
1484-
if (response.getStatus() == Response.Status.ACCEPTED.getStatusCode()) {
1485-
Thread.sleep(1000);
1486-
} else {
1487-
break;
1488-
}
1489-
}
1490-
1491-
if (response.getStatus() == Response.Status.ACCEPTED.getStatusCode()) {
1492-
LOGGER.log(Level.WARNING, "API request still not completed: {0}", response);
1493-
return response;
1494-
}
1495-
1496-
LOGGER.log(Level.FINER, "making DELETE API request to {0}", location);
1497-
Response deleteResponse = ClientBuilder.newBuilder().connectTimeout(3, TimeUnit.SECONDS).build().
1498-
target(location).request().delete();
1499-
if (deleteResponse.getStatusInfo().getFamily() != Response.Status.Family.SUCCESSFUL) {
1500-
LOGGER.log(Level.WARNING, "DELETE API call to {0} failed with HTTP error {1}",
1501-
new Object[]{location, response.getStatusInfo()});
1502-
}
1503-
1504-
return response;
1505-
}
1506-
15071461
/**
15081462
* Write the current configuration to a socket and waits for the result.
15091463
*
@@ -1524,7 +1478,7 @@ public void writeConfiguration(String host) throws IOException, InterruptedExcep
15241478
.put(Entity.xml(configXML));
15251479

15261480
if (response.getStatus() == Response.Status.ACCEPTED.getStatusCode()) {
1527-
response = waitForAsyncApi(response);
1481+
response = ApiUtils.waitForAsyncApi(response);
15281482
}
15291483

15301484
if (response.getStatusInfo().getFamily() != Response.Status.Family.SUCCESSFUL) {

‎opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@
110110
import org.opengrok.indexer.web.Util;
111111

112112
import static org.opengrok.indexer.index.IndexerUtil.getWebAppHeaders;
113+
import static org.opengrok.indexer.web.ApiUtils.waitForAsyncApi;
113114

114115
/**
115116
* This class is used to create / update the index databases. Currently we use
@@ -377,9 +378,9 @@ private void markProjectIndexed(Project project) {
377378
return;
378379
}
379380

380-
Response r;
381+
Response response;
381382
try {
382-
r = ClientBuilder.newBuilder().connectTimeout(env.getConnectTimeout(), TimeUnit.SECONDS).build()
383+
response = ClientBuilder.newBuilder().connectTimeout(env.getConnectTimeout(), TimeUnit.SECONDS).build()
383384
.target(env.getConfigURI())
384385
.path("api")
385386
.path("v1")
@@ -390,14 +391,22 @@ private void markProjectIndexed(Project project) {
390391
.headers(getWebAppHeaders())
391392
.put(Entity.text(""));
392393
} catch (RuntimeException e) {
393-
LOGGER.log(Level.WARNING, String.format("Couldn''t notify the webapp that project %s was indexed",
394+
LOGGER.log(Level.WARNING, String.format("Could not notify the webapp that project %s was indexed",
394395
project), e);
395396
return;
396397
}
397398

398-
if (r.getStatusInfo().getFamily() != Response.Status.Family.SUCCESSFUL) {
399-
LOGGER.log(Level.WARNING, "Couldn''t notify the webapp that project {0} was indexed: {1}",
400-
new Object[] {project, r});
399+
if (response.getStatus() == Response.Status.ACCEPTED.getStatusCode()) {
400+
try {
401+
response = waitForAsyncApi(response);
402+
} catch (InterruptedException e) {
403+
LOGGER.log(Level.WARNING, "interrupted while waiting for API response", e);
404+
}
405+
}
406+
407+
if (response.getStatusInfo().getFamily() != Response.Status.Family.SUCCESSFUL) {
408+
LOGGER.log(Level.WARNING, "Could not notify the webapp that project {0} was indexed: {1}",
409+
new Object[] {project, response});
401410
}
402411
}
403412

‎opengrok-indexer/src/main/java/org/opengrok/indexer/web/ApiUtils.java

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
22+
*/
23+
package org.opengrok.indexer.web;
24+
25+
import jakarta.ws.rs.client.ClientBuilder;
26+
import jakarta.ws.rs.core.HttpHeaders;
27+
import jakarta.ws.rs.core.Response;
28+
import org.jetbrains.annotations.NotNull;
29+
import org.opengrok.indexer.configuration.RuntimeEnvironment;
30+
import org.opengrok.indexer.logger.LoggerFactory;
31+
32+
import java.util.concurrent.TimeUnit;
33+
import java.util.logging.Level;
34+
import java.util.logging.Logger;
35+
36+
public class ApiUtils {
37+
38+
private static final Logger LOGGER = LoggerFactory.getLogger(ApiUtils.class);
39+
40+
private ApiUtils() {
41+
// utility class
42+
}
43+
44+
/**
45+
* Busy waits for API call to complete by repeatedly querying the status API endpoint passed
46+
* in the {@code Location} header in the response parameter. The overall time is governed
47+
* by the {@link RuntimeEnvironment#getApiTimeout()}, however each individual status check
48+
* uses {@link RuntimeEnvironment#getConnectTimeout()} so in the worst case the total time can be
49+
* {@code getApiTimeout() * getConnectTimeout()}.
50+
* @param response response returned from the server upon asynchronous API request
51+
* @return response from the status API call
52+
* @throws InterruptedException on sleep interruption
53+
* @throws IllegalArgumentException on invalid request (no {@code Location} header)
54+
*/
55+
public static @NotNull Response waitForAsyncApi(@NotNull Response response)
56+
throws InterruptedException, IllegalArgumentException {
57+
58+
String location = response.getHeaderString(HttpHeaders.LOCATION);
59+
if (location == null) {
60+
throw new IllegalArgumentException(String.format("no %s header in %s", HttpHeaders.LOCATION, response));
61+
}
62+
63+
LOGGER.log(Level.FINER, "checking asynchronous API result on {0}", location);
64+
for (int i = 0; i < RuntimeEnvironment.getInstance().getApiTimeout(); i++) {
65+
response = ClientBuilder.newBuilder().
66+
connectTimeout(RuntimeEnvironment.getInstance().getConnectTimeout(), TimeUnit.SECONDS).build().
67+
target(location).request().get();
68+
if (response.getStatus() == Response.Status.ACCEPTED.getStatusCode()) {
69+
Thread.sleep(1000);
70+
} else {
71+
break;
72+
}
73+
}
74+
75+
if (response.getStatus() == Response.Status.ACCEPTED.getStatusCode()) {
76+
LOGGER.log(Level.WARNING, "API request still not completed: {0}", response);
77+
return response;
78+
}
79+
80+
LOGGER.log(Level.FINER, "making DELETE API request to {0}", location);
81+
Response deleteResponse = ClientBuilder.newBuilder().connectTimeout(3, TimeUnit.SECONDS).build().
82+
target(location).request().delete();
83+
if (deleteResponse.getStatusInfo().getFamily() != Response.Status.Family.SUCCESSFUL) {
84+
LOGGER.log(Level.WARNING, "DELETE API call to {0} failed with HTTP error {1}",
85+
new Object[]{location, response.getStatusInfo()});
86+
}
87+
88+
return response;
89+
}
90+
}

‎opengrok-web/src/main/java/org/opengrok/web/api/v1/controller/ProjectsController.java

Lines changed: 26 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -297,9 +297,7 @@ private void deleteHistoryCacheWorkHorse(String projectName) {
297297
@PUT
298298
@Path("/{project}/indexed")
299299
@Consumes(MediaType.TEXT_PLAIN)
300-
public void markIndexed(@PathParam("project") String projectNameParam)
301-
throws ForbiddenSymlinkException, IOException, InvocationTargetException, InstantiationException,
302-
IllegalAccessException, NoSuchMethodException {
300+
public Response markIndexed(@Context HttpServletRequest request, @PathParam("project") String projectNameParam) {
303301

304302
// Avoid classification as a taint bug.
305303
final String projectName = Laundromat.launderInput(projectNameParam);
@@ -312,28 +310,34 @@ public void markIndexed(@PathParam("project") String projectNameParam)
312310

313311
project.setIndexed(true);
314312

315-
// Refresh current version of the project's repositories.
316-
List<RepositoryInfo> riList = env.getProjectRepositoriesMap().get(project);
317-
if (riList != null) {
318-
for (RepositoryInfo ri : riList) {
319-
Repository repo = getRepository(ri, CommandTimeoutType.RESTFUL);
320-
321-
if (repo != null && repo.getCurrentVersion() != null && repo.getCurrentVersion().length() > 0) {
322-
// getRepository() always creates fresh instance
323-
// of the Repository object so there is no need
324-
// to call setCurrentVersion() on it.
325-
ri.setCurrentVersion(repo.determineCurrentVersion());
326-
}
327-
}
328-
}
313+
return ApiTaskManager.getInstance().submitApiTask(PROJECTS_PATH,
314+
new ApiTask(request.getRequestURI(),
315+
() -> {
316+
// Refresh current version of the project's repositories.
317+
List<RepositoryInfo> riList = env.getProjectRepositoriesMap().get(project);
318+
if (riList != null) {
319+
for (RepositoryInfo ri : riList) {
320+
Repository repo = getRepository(ri, CommandTimeoutType.RESTFUL);
321+
322+
if (repo != null && repo.getCurrentVersion() != null &&
323+
repo.getCurrentVersion().length() > 0) {
324+
// getRepository() always creates fresh instance
325+
// of the Repository object so there is no need
326+
// to call setCurrentVersion() on it.
327+
ri.setCurrentVersion(repo.determineCurrentVersion());
328+
}
329+
}
330+
}
329331

330-
CompletableFuture.runAsync(() -> suggester.rebuild(projectName));
332+
CompletableFuture.runAsync(() -> suggester.rebuild(projectName));
331333

332-
// In case this project has just been incrementally indexed,
333-
// its IndexSearcher needs a poke.
334-
env.maybeRefreshIndexSearchers(Collections.singleton(projectName));
334+
// In case this project has just been incrementally indexed,
335+
// its IndexSearcher needs a poke.
336+
env.maybeRefreshIndexSearchers(Collections.singleton(projectName));
335337

336-
env.refreshDateForLastIndexRun();
338+
env.refreshDateForLastIndexRun();
339+
return null;
340+
}));
337341
}
338342

339343
@PUT

0 commit comments

Comments
 (0)