Skip to content

Commit 2ff3dc6

Browse files
sigramsigram
authored
SOLR-17172: Add QueryLimits termination to existing heavy SearchComponent-s (#2290)
1 parent 77f994c commit 2ff3dc6

29 files changed

+535
-94
lines changed

‎solr/CHANGES.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,9 @@ Improvements
113113

114114
* SOLR-16138: Throw a exception when issuing a streaming expression and all cores are down instead of returning 0 documents. (Antoine Bursaux via Eric Pugh)
115115

116+
* SOLR-17172: Add QueryLimits termination to the existing heavy SearchComponent-s. This allows query limits (e.g. timeAllowed,
117+
cpuAllowed) to terminate expensive operations within components if limits are exceeded. (Andrzej Bialecki)
118+
116119
Optimizations
117120
---------------------
118121
* SOLR-17144: Close searcherExecutor thread per core after 1 minute (Pierre Salagnac, Christine Poerschke)

‎solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
import org.apache.solr.search.DocListAndSet;
6262
import org.apache.solr.search.QParser;
6363
import org.apache.solr.search.QParserPlugin;
64+
import org.apache.solr.search.QueryLimits;
6465
import org.apache.solr.search.QueryParsing;
6566
import org.apache.solr.search.QueryUtils;
6667
import org.apache.solr.search.ReturnFields;
@@ -265,6 +266,8 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw
265266
}
266267
} catch (ExitableDirectoryReader.ExitingReaderException ex) {
267268
log.warn("Query: {}; ", req.getParamString(), ex);
269+
QueryLimits queryLimits = QueryLimits.getCurrentLimits();
270+
queryLimits.maybeExitWithPartialResults("MoreLikeThis");
268271
}
269272
}
270273

‎solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -237,13 +237,9 @@ public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) {
237237
rsp.setHttpCaching(httpCaching);
238238
handleRequestBody(req, rsp);
239239
// count timeouts
240-
NamedList<?> header = rsp.getResponseHeader();
241-
if (header != null) {
242-
if (Boolean.TRUE.equals(
243-
header.getBooleanArg(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY))) {
244-
metrics.numTimeouts.mark();
245-
rsp.setHttpCaching(false);
246-
}
240+
if (rsp.isPartialResults()) {
241+
metrics.numTimeouts.mark();
242+
rsp.setHttpCaching(false);
247243
}
248244
} catch (Exception e) {
249245
e = normalizeReceivedException(req, e);

‎solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@
7979
import org.apache.solr.search.DocList;
8080
import org.apache.solr.search.DocSlice;
8181
import org.apache.solr.search.QParser;
82+
import org.apache.solr.search.QueryLimits;
8283
import org.apache.solr.search.QueryUtils;
8384
import org.apache.solr.search.ReturnFields;
8485
import org.apache.solr.search.SolrIndexSearcher;
@@ -253,6 +254,10 @@ public void process(ResponseBuilder rb) throws IOException {
253254
// When no context is available we can skip the expanding
254255
return;
255256
}
257+
QueryLimits queryLimits = QueryLimits.getCurrentLimits();
258+
if (queryLimits.maybeExitWithPartialResults("Expand process")) {
259+
return;
260+
}
256261

257262
boolean nullGroupOnCurrentPage = false;
258263
int currentContext = 0;
@@ -441,6 +446,9 @@ public void process(ResponseBuilder rb) throws IOException {
441446
}
442447

443448
searcher.search(QueryUtils.combineQueryAndFilter(query, pfilter.filter), collector);
449+
if (queryLimits.maybeExitWithPartialResults("Expand expand")) {
450+
return;
451+
}
444452

445453
rb.rsp.add("expanded", groupExpandCollector.getGroups(searcher, rb.rsp.getReturnFields()));
446454
}

‎solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import org.apache.solr.schema.FieldType;
4949
import org.apache.solr.schema.PointField;
5050
import org.apache.solr.search.DocSet;
51+
import org.apache.solr.search.QueryLimits;
5152
import org.apache.solr.search.QueryParsing;
5253
import org.apache.solr.search.SyntaxError;
5354
import org.apache.solr.search.facet.FacetDebugInfo;
@@ -262,6 +263,7 @@ public List<FacetBase> getQueryFacetsForTag(String tag) {
262263
public void process(ResponseBuilder rb) throws IOException {
263264

264265
if (rb.doFacets) {
266+
QueryLimits queryLimits = QueryLimits.getCurrentLimits();
265267
SolrParams params = rb.req.getParams();
266268
SimpleFacets f = newSimpleFacets(rb.req, rb.getResults().docSet, params, rb);
267269

@@ -275,6 +277,10 @@ public void process(ResponseBuilder rb) throws IOException {
275277
}
276278

277279
NamedList<Object> counts = FacetComponent.getFacetCounts(f, fdebug);
280+
rb.rsp.add(FACET_COUNTS_KEY, counts);
281+
if (queryLimits.maybeExitWithPartialResults("Faceting counts")) {
282+
return;
283+
}
278284
String[] pivots = params.getParams(FacetParams.FACET_PIVOT);
279285
if (pivots != null && Array.getLength(pivots) != 0) {
280286
PivotFacetProcessor pivotProcessor =
@@ -283,14 +289,15 @@ public void process(ResponseBuilder rb) throws IOException {
283289
if (v != null) {
284290
counts.add(PIVOT_KEY, v);
285291
}
292+
if (queryLimits.maybeExitWithPartialResults("Faceting pivots")) {
293+
return;
294+
}
286295
}
287296

288297
if (fdebug != null) {
289298
long timeElapsed = (long) timer.getTime();
290299
fdebug.setElapse(timeElapsed);
291300
}
292-
293-
rb.rsp.add(FACET_COUNTS_KEY, counts);
294301
}
295302
}
296303

@@ -1171,6 +1178,8 @@ public void finishStage(ResponseBuilder rb) {
11711178
rb.rsp.add(FACET_COUNTS_KEY, facet_counts);
11721179

11731180
rb._facetInfo = null; // could be big, so release asap
1181+
QueryLimits queryLimits = QueryLimits.getCurrentLimits();
1182+
queryLimits.maybeExitWithPartialResults("Faceting finish");
11741183
}
11751184

11761185
private SimpleOrderedMap<List<NamedList<Object>>> createPivotFacetOutput(ResponseBuilder rb) {

‎solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import org.apache.solr.request.SolrQueryRequest;
3939
import org.apache.solr.search.QParser;
4040
import org.apache.solr.search.QParserPlugin;
41+
import org.apache.solr.search.QueryLimits;
4142
import org.apache.solr.search.QueryParsing;
4243
import org.apache.solr.search.SyntaxError;
4344
import org.apache.solr.util.SolrPluginUtils;
@@ -164,6 +165,8 @@ public void process(ResponseBuilder rb) throws IOException {
164165
// TODO ???? add this directly to the response?
165166
rb.rsp.add(highlightingResponseField(), convertHighlights(sumData));
166167
}
168+
QueryLimits queryLimits = QueryLimits.getCurrentLimits();
169+
queryLimits.maybeExitWithPartialResults("Highlighting process");
167170
}
168171
}
169172
}

‎solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
import org.apache.solr.search.DocIterator;
4848
import org.apache.solr.search.DocList;
4949
import org.apache.solr.search.DocListAndSet;
50+
import org.apache.solr.search.QueryLimits;
5051
import org.apache.solr.search.ReturnFields;
5152
import org.apache.solr.search.SolrIndexSearcher;
5253
import org.apache.solr.search.SolrReturnFields;
@@ -101,13 +102,17 @@ public void process(ResponseBuilder rb) throws IOException {
101102
MoreLikeThisHandler.MoreLikeThisHelper mlt =
102103
new MoreLikeThisHandler.MoreLikeThisHelper(params, searcher);
103104
NamedList<NamedList<?>> mltQueryByDocKey = new NamedList<>();
105+
QueryLimits queryLimits = QueryLimits.getCurrentLimits();
104106
for (DocIterator results = rb.getResults().docList.iterator(); results.hasNext(); ) {
105107
int docId = results.nextDoc();
106108
final List<MoreLikeThisHandler.InterestingTerm> interestingTerms =
107109
mlt.getInterestingTerms(mlt.getBoostedMLTQuery(docId), -1);
108110
if (interestingTerms.isEmpty()) {
109111
continue;
110112
}
113+
if (queryLimits.maybeExitWithPartialResults("MoreLikeThis process")) {
114+
break;
115+
}
111116
final String uniqueKey = rb.req.getSchema().getUniqueKeyField().getName();
112117
final Document document = rb.req.getSearcher().doc(docId);
113118
final String uniqueVal = rb.req.getSchema().printableUniqueKey(document);
@@ -416,6 +421,8 @@ NamedList<DocList> getMoreLikeThese(
416421
interestingTermsResponse = new SimpleOrderedMap<>();
417422
}
418423

424+
QueryLimits queryLimits = QueryLimits.getCurrentLimits();
425+
419426
while (iterator.hasNext()) {
420427
int id = iterator.nextDoc();
421428
int rows = p.getInt(MoreLikeThisParams.DOC_COUNT, 5);
@@ -458,6 +465,9 @@ NamedList<DocList> getMoreLikeThese(
458465
interestingTermsResponse.add(name, interestingTermsString);
459466
}
460467
}
468+
if (queryLimits.maybeExitWithPartialResults("MoreLikeThis moreLikeThese")) {
469+
break;
470+
}
461471
}
462472
// add debug information
463473
if (dbg != null) {

‎solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1361,10 +1361,9 @@ protected void returnFields(ResponseBuilder rb, ShardRequest sreq) {
13611361
if (Boolean.TRUE.equals(
13621362
responseHeader.getBooleanArg(
13631363
SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY))) {
1364-
rb.rsp
1365-
.getResponseHeader()
1366-
.asShallowMap()
1367-
.put(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
1364+
rb.rsp.setPartialResults();
1365+
rb.rsp.addPartialResponseDetail(
1366+
responseHeader.get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_DETAILS_KEY));
13681367
}
13691368
}
13701369
SolrDocumentList docs =

‎solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -445,9 +445,7 @@ public Query wrap(Query q) {
445445
public void setResult(QueryResult result) {
446446
setResults(result.getDocListAndSet());
447447
if (result.isPartialResults()) {
448-
rsp.getResponseHeader()
449-
.asShallowMap()
450-
.put(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
448+
rsp.setPartialResults();
451449
if (getResults() != null && getResults().docList == null) {
452450
getResults().docList =
453451
new DocSlice(0, 0, new int[] {}, new float[] {}, 0, 0, TotalHits.Relation.EQUAL_TO);

‎solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -497,10 +497,7 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw
497497
debug.add("explain", new NamedList<>());
498498
rb.rsp.add("debug", debug);
499499
}
500-
rb.rsp
501-
.getResponseHeader()
502-
.asShallowMap()
503-
.put(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
500+
rb.rsp.setPartialResults();
504501
}
505502
} else {
506503
// a distributed request
@@ -593,9 +590,7 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw
593590
if (allShardsFailed) {
594591
throwSolrException(srsp.getException());
595592
} else {
596-
rsp.getResponseHeader()
597-
.asShallowMap()
598-
.put(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
593+
rsp.setPartialResults();
599594
}
600595
}
601596
}

0 commit comments

Comments
 (0)