Skip to content

Commit e92cec6

Browse files
author
Vladimir Kotal
committed
introduce timeout for generating xrefs
fixes #3698
1 parent 7f430e4 commit e92cec6

File tree

6 files changed

+110
-40
lines changed

6 files changed

+110
-40
lines changed

‎opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/Ctags.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -425,8 +425,8 @@ private void addTerraformSupport(List<String> command) {
425425
* @throws IOException I/O exception
426426
* @throws InterruptedException interrupted command
427427
*/
428-
public Definitions doCtags(String file) throws IOException,
429-
InterruptedException {
428+
public Definitions doCtags(String file) throws IOException, InterruptedException {
429+
430430
if (file.length() < 1 || "\n".equals(file)) {
431431
return null;
432432
}

‎opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/plain/PlainAnalyzer.java

Lines changed: 43 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
*/
1919

2020
/*
21-
* Copyright (c) 2005, 2018, Oracle and/or its affiliates. All rights reserved.
21+
* Copyright (c) 2005, 2021, Oracle and/or its affiliates. All rights reserved.
2222
* Portions Copyright (c) 2017, 2020, Chris Fraire <[email protected]>.
2323
*/
2424
package org.opengrok.indexer.analysis.plain;
@@ -27,7 +27,12 @@
2727
import java.io.InputStream;
2828
import java.io.Reader;
2929
import java.io.Writer;
30+
import java.util.concurrent.CompletableFuture;
31+
import java.util.concurrent.ExecutionException;
32+
import java.util.concurrent.TimeUnit;
3033
import java.util.function.Supplier;
34+
import java.util.logging.Level;
35+
import java.util.logging.Logger;
3136

3237
import org.apache.lucene.document.Document;
3338
import org.apache.lucene.document.StoredField;
@@ -44,6 +49,8 @@
4449
import org.opengrok.indexer.analysis.TextAnalyzer;
4550
import org.opengrok.indexer.analysis.WriteXrefArgs;
4651
import org.opengrok.indexer.analysis.Xrefer;
52+
import org.opengrok.indexer.configuration.RuntimeEnvironment;
53+
import org.opengrok.indexer.logger.LoggerFactory;
4754
import org.opengrok.indexer.search.QueryBuilder;
4855
import org.opengrok.indexer.util.NullWriter;
4956

@@ -55,6 +62,8 @@
5562
*/
5663
public class PlainAnalyzer extends TextAnalyzer {
5764

65+
private static final Logger LOGGER = LoggerFactory.getLogger(PlainAnalyzer.class);
66+
5867
/**
5968
* Creates a new instance of PlainAnalyzer.
6069
* @param factory defined instance for the analyzer
@@ -108,17 +117,15 @@ protected Reader getReader(InputStream stream) throws IOException {
108117
}
109118

110119
@Override
111-
public void analyze(Document doc, StreamSource src, Writer xrefOut)
112-
throws IOException, InterruptedException {
120+
public void analyze(Document doc, StreamSource src, Writer xrefOut) throws IOException, InterruptedException {
113121
Definitions defs = null;
114122
NullWriter nullWriter = null;
115123

116-
doc.add(new OGKTextField(QueryBuilder.FULL,
117-
getReader(src.getStream())));
124+
doc.add(new OGKTextField(QueryBuilder.FULL, getReader(src.getStream())));
118125

119-
String fullpath = doc.get(QueryBuilder.FULLPATH);
120-
if (fullpath != null && ctags != null) {
121-
defs = ctags.doCtags(fullpath);
126+
String fullPath = doc.get(QueryBuilder.FULLPATH);
127+
if (fullPath != null && ctags != null) {
128+
defs = ctags.doCtags(fullPath);
122129
if (defs != null && defs.numberOfSymbols() > 0) {
123130
tryAddingDefs(doc, defs, src);
124131
byte[] tags = defs.serialize();
@@ -137,7 +144,7 @@ public void analyze(Document doc, StreamSource src, Writer xrefOut)
137144
if (scopesEnabled && xrefOut == null) {
138145
/*
139146
* Scopes are generated during xref generation. If xrefs are
140-
* turned off we still need to run writeXref to produce scopes,
147+
* turned off we still need to run writeXref() to produce scopes,
141148
* we use a dummy writer that will throw away any xref output.
142149
*/
143150
nullWriter = new NullWriter();
@@ -146,20 +153,37 @@ public void analyze(Document doc, StreamSource src, Writer xrefOut)
146153

147154
if (xrefOut != null) {
148155
try (Reader in = getReader(src.getStream())) {
156+
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
149157
WriteXrefArgs args = new WriteXrefArgs(in, xrefOut);
150158
args.setDefs(defs);
151159
args.setProject(project);
152-
Xrefer xref = writeXref(args);
153-
154-
Scopes scopes = xref.getScopes();
155-
if (scopes.size() > 0) {
156-
byte[] scopesSerialized = scopes.serialize();
157-
doc.add(new StoredField(QueryBuilder.SCOPES,
158-
scopesSerialized));
160+
CompletableFuture<Xrefer> future = CompletableFuture.supplyAsync(() -> {
161+
try {
162+
return writeXref(args);
163+
} catch (IOException e) {
164+
LOGGER.log(Level.WARNING, "I/O exception when generating xref for " + fullPath, e);
165+
return null;
166+
}
167+
}, env.getIndexerParallelizer().getXrefWatcherExecutor()).
168+
orTimeout(env.getXrefTimeout(), TimeUnit.SECONDS);
169+
Xrefer xref = future.get(); // Will throw ExecutionException wrapping TimeoutException on timeout.
170+
171+
if (xref != null) {
172+
Scopes scopes = xref.getScopes();
173+
if (scopes.size() > 0) {
174+
byte[] scopesSerialized = scopes.serialize();
175+
doc.add(new StoredField(QueryBuilder.SCOPES,
176+
scopesSerialized));
177+
}
178+
179+
String path = doc.get(QueryBuilder.PATH);
180+
addNumLinesLOC(doc, new NumLinesLOC(path, xref.getLineNumber(), xref.getLOC()));
181+
} else {
182+
// Re-throw the exception from writeXref(). We no longer have the original exception, though.
183+
throw new IOException("I/O exception when generating xref for " + fullPath);
159184
}
160-
161-
String path = doc.get(QueryBuilder.PATH);
162-
addNumLinesLOC(doc, new NumLinesLOC(path, xref.getLineNumber(), xref.getLOC()));
185+
} catch (ExecutionException e) {
186+
throw new InterruptedException("failed to generate xref :" + e);
163187
} finally {
164188
if (nullWriter != null) {
165189
nullWriter.close();

‎opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/Configuration.java

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,7 @@ public final class Configuration {
214214
private int webappStartCommandTimeout; // in seconds
215215
private int restfulCommandTimeout; // in seconds
216216
private long ctagsTimeout; // in seconds
217+
private long xrefTimeout; // in seconds
217218
private boolean scopesEnabled;
218219
private boolean projectsEnabled;
219220
private boolean foldingEnabled;
@@ -475,6 +476,24 @@ public void setCtagsTimeout(long timeout) throws IllegalArgumentException {
475476
this.ctagsTimeout = timeout;
476477
}
477478

479+
public long getXrefTimeout() {
480+
return xrefTimeout;
481+
}
482+
483+
/**
484+
* Set the timeout for generating xrefs to a new value.
485+
*
486+
* @param timeout the new value
487+
* @throws IllegalArgumentException when the timeout is negative
488+
*/
489+
public void setXrefTimeout(long timeout) throws IllegalArgumentException {
490+
if (timeout < 0) {
491+
throw new IllegalArgumentException(
492+
String.format(NEGATIVE_NUMBER_ERROR, "xrefTimeout", timeout));
493+
}
494+
this.xrefTimeout = timeout;
495+
}
496+
478497
public boolean isLastEditedDisplayMode() {
479498
return lastEditedDisplayMode;
480499
}
@@ -569,11 +588,11 @@ public Configuration() {
569588
//setTabSize(4);
570589
setTagsEnabled(false);
571590
//setUserPage("http://www.myserver.org/viewProfile.jspa?username=");
572-
// Set to empty string so we can append it to the URL
573-
// unconditionally later.
591+
// Set to empty string so we can append it to the URL unconditionally later.
574592
setUserPageSuffix("");
575593
setWebappLAF("default");
576594
// webappCtags is default(boolean)
595+
setXrefTimeout(30);
577596
}
578597

579598
public String getRepoCmd(String clazzName) {

‎opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -298,6 +298,14 @@ public void setCtagsTimeout(long timeout) {
298298
syncWriteConfiguration(timeout, Configuration::setCtagsTimeout);
299299
}
300300

301+
public long getXrefTimeout() {
302+
return syncReadConfiguration(Configuration::getXrefTimeout);
303+
}
304+
305+
public void setXrefTimeout(long timeout) {
306+
syncWriteConfiguration(timeout, Configuration::setXrefTimeout);
307+
}
308+
301309
public void setLastEditedDisplayMode(boolean lastEditedDisplayMode) {
302310
syncWriteConfiguration(lastEditedDisplayMode, Configuration::setLastEditedDisplayMode);
303311
}

‎opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java

Lines changed: 8 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1330,8 +1330,7 @@ private void indexParallel(String dir, IndexDownArgs args) {
13301330
AtomicInteger successCounter = new AtomicInteger();
13311331
AtomicInteger currentCounter = new AtomicInteger();
13321332
AtomicInteger alreadyClosedCounter = new AtomicInteger();
1333-
IndexerParallelizer parallelizer = RuntimeEnvironment.getInstance().
1334-
getIndexerParallelizer();
1333+
IndexerParallelizer parallelizer = RuntimeEnvironment.getInstance().getIndexerParallelizer();
13351334
ObjectPool<Ctags> ctagsPool = parallelizer.getCtagsPool();
13361335

13371336
Map<Boolean, List<IndexFileWork>> bySuccess = null;
@@ -1355,8 +1354,7 @@ private void indexParallel(String dir, IndexDownArgs args) {
13551354
}
13561355
} catch (AlreadyClosedException e) {
13571356
alreadyClosedCounter.incrementAndGet();
1358-
String errmsg = String.format("ERROR addFile(): %s",
1359-
x.file);
1357+
String errmsg = String.format("ERROR addFile(): %s", x.file);
13601358
LOGGER.log(Level.SEVERE, errmsg, e);
13611359
x.exception = e;
13621360
ret = false;
@@ -1369,8 +1367,7 @@ private void indexParallel(String dir, IndexDownArgs args) {
13691367
x.exception = e;
13701368
ret = false;
13711369
} catch (RuntimeException | IOException e) {
1372-
String errmsg = String.format("ERROR addFile(): %s",
1373-
x.file);
1370+
String errmsg = String.format("ERROR addFile(): %s", x.file);
13741371
LOGGER.log(Level.WARNING, errmsg, e);
13751372
x.exception = e;
13761373
ret = false;
@@ -1390,8 +1387,7 @@ private void indexParallel(String dir, IndexDownArgs args) {
13901387
} catch (InterruptedException | ExecutionException e) {
13911388
int successCount = successCounter.intValue();
13921389
double successPct = 100.0 * successCount / worksCount;
1393-
String exmsg = String.format(
1394-
"%d successes (%.1f%%) after aborting parallel-indexing",
1390+
String exmsg = String.format("%d successes (%.1f%%) after aborting parallel-indexing",
13951391
successCount, successPct);
13961392
LOGGER.log(Level.SEVERE, exmsg, e);
13971393
}
@@ -1401,28 +1397,24 @@ private void indexParallel(String dir, IndexDownArgs args) {
14011397
// Start with failureCount=worksCount, and then subtract successes.
14021398
int failureCount = worksCount;
14031399
if (bySuccess != null) {
1404-
List<IndexFileWork> successes = bySuccess.getOrDefault(
1405-
Boolean.TRUE, null);
1400+
List<IndexFileWork> successes = bySuccess.getOrDefault(Boolean.TRUE, null);
14061401
if (successes != null) {
14071402
failureCount -= successes.size();
14081403
}
14091404
}
14101405
if (failureCount > 0) {
14111406
double pctFailed = 100.0 * failureCount / worksCount;
1412-
String exmsg = String.format(
1413-
"%d failures (%.1f%%) while parallel-indexing",
1414-
failureCount, pctFailed);
1407+
String exmsg = String.format("%d failures (%.1f%%) while parallel-indexing", failureCount, pctFailed);
14151408
LOGGER.log(Level.WARNING, exmsg);
14161409
}
14171410

1418-
/**
1411+
/*
14191412
* Encountering an AlreadyClosedException is severe enough to abort the
14201413
* run, since it will fail anyway later upon trying to commit().
14211414
*/
14221415
int numAlreadyClosed = alreadyClosedCounter.get();
14231416
if (numAlreadyClosed > 0) {
1424-
throw new AlreadyClosedException(String.format("count=%d",
1425-
numAlreadyClosed));
1417+
throw new AlreadyClosedException(String.format("count=%d", numAlreadyClosed));
14261418
}
14271419
}
14281420

‎opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexerParallelizer.java

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
/*
2121
* Copyright (c) 2017, 2020, Chris Fraire <[email protected]>.
22-
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
22+
* Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
2323
*/
2424
package org.opengrok.indexer.index;
2525

@@ -61,6 +61,7 @@ public class IndexerParallelizer implements AutoCloseable {
6161
private LazilyInstantiate<ExecutorService> lzHistoryExecutor;
6262
private LazilyInstantiate<ExecutorService> lzHistoryFileExecutor;
6363
private LazilyInstantiate<ExecutorService> lzCtagsWatcherExecutor;
64+
private LazilyInstantiate<ExecutorService> lzXrefWatcherExecutor;
6465

6566
/**
6667
* Initializes a new instance using settings from the specified environment
@@ -84,6 +85,7 @@ public IndexerParallelizer(RuntimeEnvironment env) {
8485
createLazyHistoryExecutor();
8586
createLazyHistoryFileExecutor();
8687
createLazyCtagsWatcherExecutor();
88+
createLazyXrefWatcherExecutor();
8789
}
8890

8991
/**
@@ -128,6 +130,13 @@ public ExecutorService getCtagsWatcherExecutor() {
128130
return lzCtagsWatcherExecutor.get();
129131
}
130132

133+
/**
134+
* @return the Executor used for enforcing xref timeouts.
135+
*/
136+
public ExecutorService getXrefWatcherExecutor() {
137+
return lzXrefWatcherExecutor.get();
138+
}
139+
131140
/**
132141
* Calls {@link #bounce()}, which prepares for -- but does not start -- new
133142
* pools.
@@ -160,6 +169,7 @@ public void bounce() {
160169
bounceHistoryExecutor();
161170
bounceHistoryRenamedExecutor();
162171
bounceCtagsWatcherExecutor();
172+
bounceXrefWatcherExecutor();
163173
}
164174

165175
private void bounceForkJoinPool() {
@@ -210,6 +220,14 @@ private void bounceCtagsWatcherExecutor() {
210220
}
211221
}
212222

223+
private void bounceXrefWatcherExecutor() {
224+
if (lzXrefWatcherExecutor.isActive()) {
225+
ExecutorService formerXrefWatcherExecutor = lzXrefWatcherExecutor.get();
226+
createLazyXrefWatcherExecutor();
227+
formerXrefWatcherExecutor.shutdown();
228+
}
229+
}
230+
213231
private void createLazyForkJoinPool() {
214232
lzForkJoinPool = LazilyInstantiate.using(() ->
215233
new ForkJoinPool(indexingParallelism));
@@ -230,6 +248,15 @@ private void createLazyCtagsWatcherExecutor() {
230248
}));
231249
}
232250

251+
private void createLazyXrefWatcherExecutor() {
252+
lzXrefWatcherExecutor = LazilyInstantiate.using(() ->
253+
new ScheduledThreadPoolExecutor(1, runnable -> {
254+
Thread thread = Executors.defaultThreadFactory().newThread(runnable);
255+
thread.setName("xref-watcher-" + thread.getId());
256+
return thread;
257+
}));
258+
}
259+
233260
private void createLazyFixedExecutor() {
234261
lzFixedExecutor = LazilyInstantiate.using(() ->
235262
Executors.newFixedThreadPool(indexingParallelism));

0 commit comments

Comments
 (0)