Skip to content

Commit 5e328ee

Browse files
mlbiscocMatthew Biscocho
mlbiscoc
and
Matthew Biscocho
authored
SOLR-17582: Stream CLUSTERSTATUS API for SolrJ version >= 9.9 (#3156)
Or if unknown (e.g. some misc. HTTP/JSON). Co-authored-by: Matthew Biscocho <[email protected]>
1 parent b779ed0 commit 5e328ee

File tree

6 files changed

+110
-46
lines changed

6 files changed

+110
-46
lines changed

‎solr/core/src/java/org/apache/solr/cli/StatusTool.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -353,7 +353,7 @@ protected Map<String, String> getCloudStatus(SolrClient solrClient, String zkHos
353353
cloudStatus.put("liveNodes", String.valueOf(liveNodes.size()));
354354

355355
// TODO get this as a metric from the metrics API instead, or something else.
356-
var collections = (NamedList<Object>) json.findRecursive("cluster", "collections");
356+
var collections = (Map<String, Object>) json.findRecursive("cluster", "collections");
357357
cloudStatus.put("collections", String.valueOf(collections.size()));
358358

359359
return cloudStatus;

‎solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java‎

Lines changed: 18 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import java.util.Objects;
2727
import java.util.Set;
2828
import java.util.stream.Stream;
29+
import org.apache.solr.client.api.util.SolrVersion;
2930
import org.apache.solr.common.MapWriter;
3031
import org.apache.solr.common.SolrException;
3132
import org.apache.solr.common.cloud.Aliases;
@@ -34,7 +35,6 @@
3435
import org.apache.solr.common.cloud.PerReplicaStates;
3536
import org.apache.solr.common.cloud.Replica;
3637
import org.apache.solr.common.cloud.ZkStateReader;
37-
import org.apache.solr.common.params.CommonParams;
3838
import org.apache.solr.common.params.ShardParams;
3939
import org.apache.solr.common.params.SolrParams;
4040
import org.apache.solr.common.util.NamedList;
@@ -101,7 +101,7 @@ public ClusterStatus(ZkStateReader zkStateReader, SolrParams params) {
101101
collection = params.get(ZkStateReader.COLLECTION_PROP);
102102
}
103103

104-
public void getClusterStatus(NamedList<Object> results)
104+
public void getClusterStatus(NamedList<Object> results, SolrVersion solrVersion)
105105
throws KeeperException, InterruptedException {
106106
NamedList<Object> clusterStatus = new SimpleOrderedMap<>();
107107

@@ -127,7 +127,7 @@ public void getClusterStatus(NamedList<Object> results)
127127

128128
if (withCollection) {
129129
assert liveNodes != null;
130-
fetchClusterStatusForCollOrAlias(clusterStatus, liveNodes, aliases);
130+
fetchClusterStatusForCollOrAlias(clusterStatus, liveNodes, aliases, solrVersion);
131131
}
132132

133133
if (withAliases) {
@@ -158,7 +158,10 @@ public void getClusterStatus(NamedList<Object> results)
158158
}
159159

160160
private void fetchClusterStatusForCollOrAlias(
161-
NamedList<Object> clusterStatus, List<String> liveNodes, Aliases aliases) {
161+
NamedList<Object> clusterStatus,
162+
List<String> liveNodes,
163+
Aliases aliases,
164+
SolrVersion solrVersion) {
162165

163166
// read aliases
164167
Map<String, List<String>> collectionVsAliases = new HashMap<>();
@@ -206,19 +209,7 @@ private void fetchClusterStatusForCollOrAlias(
206209
}
207210
}
208211

209-
// Because of back-compat for SolrJ, create the whole response into a NamedList
210-
// Otherwise stream with MapWriter to save memory
211-
if (CommonParams.JAVABIN.equals(solrParams.get(CommonParams.WT))) {
212-
NamedList<Object> collectionProps = new SimpleOrderedMap<>();
213-
collectionStream.forEach(
214-
collectionState -> {
215-
collectionProps.add(
216-
collectionState.getName(),
217-
buildResponseForCollection(
218-
collectionState, collectionVsAliases, routeKey, liveNodes, requestedShards));
219-
});
220-
clusterStatus.add("collections", collectionProps);
221-
} else {
212+
if (solrVersion == null || solrVersion.greaterThanOrEqualTo(SolrVersion.valueOf("9.9.0"))) {
222213
MapWriter collectionPropsWriter =
223214
ew -> {
224215
collectionStream.forEach(
@@ -234,6 +225,16 @@ private void fetchClusterStatusForCollOrAlias(
234225
});
235226
};
236227
clusterStatus.add("collections", collectionPropsWriter);
228+
} else {
229+
NamedList<Object> collectionProps = new SimpleOrderedMap<>();
230+
collectionStream.forEach(
231+
collectionState -> {
232+
collectionProps.add(
233+
collectionState.getName(),
234+
buildResponseForCollection(
235+
collectionState, collectionVsAliases, routeKey, liveNodes, requestedShards));
236+
});
237+
clusterStatus.add("collections", collectionProps);
237238
}
238239
}
239240

‎solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -975,7 +975,7 @@ public Map<String, Object> execute(
975975
CLUSTERSTATUS,
976976
(req, rsp, h) -> {
977977
new ClusterStatus(h.coreContainer.getZkController().getZkStateReader(), req.getParams())
978-
.getClusterStatus(rsp.getValues());
978+
.getClusterStatus(rsp.getValues(), req.getHttpSolrCall().getUserAgentSolrVersion());
979979
return null;
980980
}),
981981
ADDREPLICAPROP_OP(

‎solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java‎

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -131,10 +131,11 @@ private void testModifyCollection() throws Exception {
131131
.getResponse();
132132
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
133133
assertNotNull("Cluster state should not be null", cluster);
134-
NamedList<?> collections = (NamedList<?>) cluster.get("collections");
134+
Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
135135
assertNotNull("Collections should not be null in cluster state", collections);
136136
assertEquals(1, collections.size());
137-
assertEquals("25", collections._getStr(List.of(COLLECTION_NAME, "replicationFactor"), null));
137+
Map<?, ?> collectionProperties = (Map<?, ?>) collections.get(COLLECTION_NAME);
138+
assertEquals("25", collectionProperties.get("replicationFactor"));
138139

139140
params = new ModifiableSolrParams();
140141
params.set("action", CollectionParams.CollectionAction.MODIFYCOLLECTION.toString());
@@ -153,10 +154,11 @@ private void testModifyCollection() throws Exception {
153154
System.out.println(rsp);
154155
cluster = (NamedList<?>) rsp.get("cluster");
155156
assertNotNull("Cluster state should not be null", cluster);
156-
collections = (NamedList<?>) cluster.get("collections");
157+
collections = (Map<?, ?>) cluster.get("collections");
157158
assertNotNull("Collections should not be null in cluster state", collections);
158159
assertEquals(1, collections.size());
159-
assertNull(collections._getStr(List.of(COLLECTION_NAME, "replicationFactor"), null));
160+
collectionProperties = (Map<?, ?>) collections.get(COLLECTION_NAME);
161+
assertNull(collectionProperties.get("replicationFactor"));
160162

161163
params = new ModifiableSolrParams();
162164
params.set("action", CollectionParams.CollectionAction.MODIFYCOLLECTION.toString());
@@ -255,7 +257,7 @@ private void testNoConfigset() throws Exception {
255257
NamedList<?> rsp = client.request(req);
256258
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
257259
assertNotNull("Cluster state should not be null", cluster);
258-
NamedList<?> collections = (NamedList<?>) cluster.get("collections");
260+
Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
259261
assertNotNull("Collections should not be null in cluster state", collections);
260262
assertNotNull(
261263
"Testing to insure collections are returned", collections.get(COLLECTION_NAME1));
@@ -282,7 +284,7 @@ private void assertCountsForRepFactorAndNrtReplicas(CloudSolrClient client, Stri
282284
NamedList<Object> rsp = client.request(request);
283285
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
284286
assertNotNull("Cluster state should not be null", cluster);
285-
NamedList<?> collections = (NamedList<?>) cluster.get("collections");
287+
Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
286288
assertNotNull("Collections should not be null in cluster state", collections);
287289
assertEquals(1, collections.size());
288290
@SuppressWarnings({"unchecked"})
@@ -304,7 +306,7 @@ private void clusterStatusWithCollectionAndShard() throws IOException, SolrServe
304306
NamedList<Object> rsp = client.request(request);
305307
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
306308
assertNotNull("Cluster state should not be null", cluster);
307-
NamedList<?> collections = (NamedList<?>) cluster.get("collections");
309+
Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
308310
assertNotNull("Collections should not be null in cluster state", collections);
309311
assertNotNull(collections.get(COLLECTION_NAME));
310312
assertEquals(1, collections.size());
@@ -330,7 +332,7 @@ private void clusterStatusWithCollectionAndMultipleShards()
330332
NamedList<Object> rsp = request.process(client).getResponse();
331333
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
332334
assertNotNull("Cluster state should not be null", cluster);
333-
NamedList<?> collections = (NamedList<?>) cluster.get("collections");
335+
Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
334336
assertNotNull("Collections should not be null in cluster state", collections);
335337
assertNotNull(collections.get(COLLECTION_NAME));
336338
assertEquals(1, collections.size());
@@ -465,7 +467,7 @@ private void clusterStatusNoCollection() throws Exception {
465467
NamedList<Object> rsp = client.request(request);
466468
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
467469
assertNotNull("Cluster state should not be null", cluster);
468-
NamedList<?> collections = (NamedList<?>) cluster.get("collections");
470+
Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
469471
assertNotNull("Collections should not be null in cluster state", collections);
470472
assertNotNull(collections.get(COLLECTION_NAME1));
471473
assertEquals(4, collections.size());
@@ -487,7 +489,7 @@ private void clusterStatusWithCollection() throws IOException, SolrServerExcepti
487489
NamedList<Object> rsp = client.request(request);
488490
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
489491
assertNotNull("Cluster state should not be null", cluster);
490-
NamedList<?> collections = (NamedList<?>) cluster.get("collections");
492+
Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
491493
assertNotNull("Collections should not be null in cluster state", collections);
492494
assertEquals(1, collections.size());
493495
@SuppressWarnings({"unchecked"})
@@ -517,7 +519,7 @@ private void clusterStatusZNodeVersion() throws Exception {
517519
NamedList<Object> rsp = client.request(request);
518520
NamedList<Object> cluster = (NamedList<Object>) rsp.get("cluster");
519521
assertNotNull("Cluster state should not be null", cluster);
520-
NamedList<Object> collections = (NamedList<Object>) cluster.get("collections");
522+
Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
521523
assertNotNull("Collections should not be null in cluster state", collections);
522524
assertEquals(1, collections.size());
523525
Map<String, Object> collection = (Map<String, Object>) collections.get(cname);
@@ -533,7 +535,7 @@ private void clusterStatusZNodeVersion() throws Exception {
533535

534536
rsp = client.request(request);
535537
cluster = (NamedList<Object>) rsp.get("cluster");
536-
collections = (NamedList<Object>) cluster.get("collections");
538+
collections = (Map<?, ?>) cluster.get("collections");
537539
collection = (Map<String, Object>) collections.get(cname);
538540
Integer newVersion = (Integer) collection.get("znodeVersion");
539541
assertNotNull(newVersion);
@@ -560,7 +562,7 @@ private void clusterStatusWithRouteKey() throws IOException, SolrServerException
560562
NamedList<Object> cluster = (NamedList<Object>) rsp.get("cluster");
561563
assertNotNull("Cluster state should not be null", cluster);
562564
@SuppressWarnings({"unchecked"})
563-
NamedList<Object> collections = (NamedList<Object>) cluster.get("collections");
565+
Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
564566
assertNotNull("Collections should not be null in cluster state", collections);
565567
assertNotNull(collections.get(DEFAULT_COLLECTION));
566568
assertEquals(1, collections.size());
@@ -607,7 +609,7 @@ private void clusterStatusAliasTest() throws Exception {
607609
DEFAULT_COLLECTION + "," + COLLECTION_NAME,
608610
aliases.get("myalias"));
609611

610-
NamedList<Object> collections = (NamedList<Object>) cluster.get("collections");
612+
Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
611613
assertNotNull("Collections should not be null in cluster state", collections);
612614
assertNotNull(collections.get(DEFAULT_COLLECTION));
613615
Map<String, Object> collection = (Map<String, Object>) collections.get(DEFAULT_COLLECTION);
@@ -627,7 +629,7 @@ private void clusterStatusAliasTest() throws Exception {
627629

628630
cluster = (NamedList<Object>) rsp.get("cluster");
629631
assertNotNull("Cluster state should not be null", cluster);
630-
collections = (NamedList<Object>) cluster.get("collections");
632+
collections = (Map<?, ?>) cluster.get("collections");
631633
assertNotNull("Collections should not be null in cluster state", collections);
632634
assertNotNull(collections.get(DEFAULT_COLLECTION));
633635
assertNotNull(collections.get(COLLECTION_NAME));

‎solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
import java.util.Collections;
3030
import java.util.List;
3131
import java.util.Map;
32-
import java.util.Map.Entry;
3332
import java.util.Set;
3433
import java.util.concurrent.TimeUnit;
3534
import java.util.stream.Collectors;
@@ -46,7 +45,6 @@
4645
import org.apache.solr.common.params.ModifiableSolrParams;
4746
import org.apache.solr.common.util.CollectionUtil;
4847
import org.apache.solr.common.util.EnvUtils;
49-
import org.apache.solr.common.util.NamedList;
5048
import org.apache.solr.common.util.SimpleOrderedMap;
5149
import org.apache.solr.common.util.URLUtil;
5250
import org.apache.solr.common.util.Utils;
@@ -160,15 +158,12 @@ private ClusterState fetchClusterState(SolrClient client)
160158
liveNodesTimestamp = System.nanoTime();
161159
}
162160

163-
var collectionsNl = (NamedList<Map<String, Object>>) cluster.get("collections");
161+
var collectionsMap = (Map<String, Map<String, Object>>) cluster.get("collections");
164162

165163
Map<String, DocCollection> collStateByName =
166-
CollectionUtil.newLinkedHashMap(collectionsNl.size());
167-
for (Entry<String, Map<String, Object>> entry : collectionsNl) {
168-
collStateByName.put(
169-
entry.getKey(), getDocCollectionFromObjects(entry.getKey(), entry.getValue()));
170-
}
171-
164+
CollectionUtil.newLinkedHashMap(collectionsMap.size());
165+
collectionsMap.forEach(
166+
(key, value) -> collStateByName.put(key, getDocCollectionFromObjects(key, value)));
172167
return new ClusterState(this.liveNodes, collStateByName);
173168
}
174169

@@ -205,7 +200,7 @@ private DocCollection fetchCollectionState(SolrClient client, String collection)
205200
SimpleOrderedMap<?> cluster =
206201
submitClusterStateRequest(client, collection, ClusterStateRequestType.FETCH_COLLECTION);
207202

208-
var collStateMap = (Map<String, Object>) cluster.findRecursive("collections", collection);
203+
var collStateMap = (Map<String, Object>) cluster._get(List.of("collections", collection), null);
209204
if (collStateMap == null) {
210205
throw new NotACollectionException(); // probably an alias
211206
}

‎solr/solrj/src/test/org/apache/solr/client/solrj/impl/ClusterStateProviderTest.java‎

Lines changed: 68 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323

2424
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
2525
import java.io.IOException;
26+
import java.lang.invoke.MethodHandles;
2627
import java.lang.reflect.InvocationTargetException;
2728
import java.time.Instant;
2829
import java.util.List;
@@ -36,6 +37,8 @@
3637
import org.apache.solr.common.cloud.ClusterState;
3738
import org.apache.solr.common.cloud.DocCollection;
3839
import org.apache.solr.common.util.NamedList;
40+
import org.eclipse.jetty.http.HttpField;
41+
import org.eclipse.jetty.http.HttpHeader;
3942
import org.hamcrest.Matchers;
4043
import org.junit.After;
4144
import org.junit.BeforeClass;
@@ -135,7 +138,6 @@ public void testGetState() throws Exception {
135138
try (ClusterStateProvider provider = createClusterStateProvider()) {
136139

137140
ClusterState.CollectionRef collectionRef = provider.getState("testGetState");
138-
139141
DocCollection docCollection = collectionRef.get();
140142
assertNotNull(docCollection);
141143
assertEquals(
@@ -159,7 +161,7 @@ private Instant getCreationTimeFromClusterStatus(String collectionName)
159161
NamedList<Object> response = clusterStatusResponse.getResponse();
160162

161163
NamedList<Object> cluster = (NamedList<Object>) response.get("cluster");
162-
NamedList<Object> collections = (NamedList<Object>) cluster.get("collections");
164+
Map<String, Object> collections = (Map<String, Object>) cluster.get("collections");
163165
Map<String, Object> collection = (Map<String, Object>) collections.get(collectionName);
164166
return Instant.ofEpochMilli((long) collection.get("creationTimeMillis"));
165167
}
@@ -198,6 +200,70 @@ public void testClusterStateProvider() throws SolrServerException, IOException {
198200
}
199201
}
200202

203+
@Test
204+
public void testClusterStateProviderOldVersion() throws SolrServerException, IOException {
205+
CollectionAdminRequest.setClusterProperty("ext.foo", "bar").process(cluster.getSolrClient());
206+
createCollection("col1");
207+
createCollection("col2");
208+
209+
try (var cspZk = zkClientClusterStateProvider();
210+
var cspHttp = http2ClusterStateProvider()) {
211+
// SolrJ < version 9.9.0 for non streamed response
212+
cspHttp
213+
.getHttpClient()
214+
.getHttpClient()
215+
.setUserAgentField(
216+
new HttpField(
217+
HttpHeader.USER_AGENT,
218+
"Solr[" + MethodHandles.lookup().lookupClass().getName() + "] " + "9.8.0"));
219+
220+
assertThat(cspHttp.getCollection("col1"), equalTo(cspZk.getCollection("col1")));
221+
222+
final var clusterStateZk = cspZk.getClusterState();
223+
final var clusterStateHttp = cspHttp.getClusterState();
224+
assertThat(
225+
clusterStateHttp.getLiveNodes(),
226+
containsInAnyOrder(clusterStateHttp.getLiveNodes().toArray()));
227+
assertEquals(2, clusterStateZk.size());
228+
assertEquals(clusterStateZk.size(), clusterStateHttp.size());
229+
assertThat(
230+
clusterStateHttp.collectionStream().toList(),
231+
containsInAnyOrder(clusterStateHttp.collectionStream().toArray()));
232+
233+
assertThat(
234+
clusterStateZk.getCollection("col2"), equalTo(clusterStateHttp.getCollection("col2")));
235+
}
236+
}
237+
238+
@Test
239+
public void testClusterStateProviderEmptySolrVersion() throws SolrServerException, IOException {
240+
CollectionAdminRequest.setClusterProperty("ext.foo", "bar").process(cluster.getSolrClient());
241+
createCollection("col1");
242+
createCollection("col2");
243+
244+
try (var cspZk = zkClientClusterStateProvider();
245+
var cspHttp = http2ClusterStateProvider()) {
246+
247+
cspHttp.getHttpClient().getHttpClient().setUserAgentField(null);
248+
249+
assertThat(cspHttp.getCollection("col1"), equalTo(cspZk.getCollection("col1")));
250+
251+
final var clusterStateZk = cspZk.getClusterState();
252+
final var clusterStateHttp = cspHttp.getClusterState();
253+
assertThat(
254+
clusterStateHttp.getLiveNodes(),
255+
containsInAnyOrder(clusterStateHttp.getLiveNodes().toArray()));
256+
assertEquals(2, clusterStateZk.size());
257+
assertEquals(clusterStateZk.size(), clusterStateHttp.size());
258+
assertThat(
259+
clusterStateHttp.collectionStream().toList(),
260+
containsInAnyOrder(clusterStateHttp.collectionStream().toArray()));
261+
262+
assertThat(
263+
clusterStateZk.getCollection("col2"), equalTo(clusterStateHttp.getCollection("col2")));
264+
}
265+
}
266+
201267
@Test
202268
public void testClusterStateProviderDownedInitialLiveNodes() throws Exception {
203269
try (var cspHttp = http2ClusterStateProvider()) {

0 commit comments

Comments
 (0)