diff --git a/changelog/unreleased/SOLR-16458-migrate-node-health-api.yml b/changelog/unreleased/SOLR-16458-migrate-node-health-api.yml new file mode 100644 index 000000000000..59dc710934dd --- /dev/null +++ b/changelog/unreleased/SOLR-16458-migrate-node-health-api.yml @@ -0,0 +1,8 @@ +title: "SolrJ now offers a SolrRequest class allowing users to perform v2 single-node healthchecks: NodeApi.Healthcheck" +type: added +authors: + - name: Eric Pugh + - name: Jason Gerlowski +links: + - name: SOLR-16458 + url: https://issues.apache.org/jira/browse/SOLR-16458 diff --git a/changelog/unreleased/SOLR-17973-fix-shards-preference-cross-collection-join.yml b/changelog/unreleased/SOLR-17973-fix-shards-preference-cross-collection-join.yml new file mode 100644 index 000000000000..1620764ea063 --- /dev/null +++ b/changelog/unreleased/SOLR-17973-fix-shards-preference-cross-collection-join.yml @@ -0,0 +1,7 @@ +title: "SOLR-17973: Fix `shards.preference` not respected for cross-collection join queries" +type: fixed +authors: + - name: khushjain +links: + - name: SOLR-17973 + url: https://issues.apache.org/jira/browse/SOLR-17973 diff --git a/changelog/unreleased/SOLR-18155-election-leak.yml b/changelog/unreleased/SOLR-18155-election-leak.yml new file mode 100644 index 000000000000..a43436d09805 --- /dev/null +++ b/changelog/unreleased/SOLR-18155-election-leak.yml @@ -0,0 +1,7 @@ +title: Abort shard leader election if container shutdown sequence has started, so we don't have leaders elected very late and not properly closed. +type: fixed +authors: + - name: Pierre Salagnac +links: + - name: SOLR-18155 + url: https://issues.apache.org/jira/browse/SOLR-18155 diff --git a/changelog/unreleased/SOLR-18159-physical-memory-metrics.yml b/changelog/unreleased/SOLR-18159-physical-memory-metrics.yml new file mode 100644 index 000000000000..c24a17cd3e3b --- /dev/null +++ b/changelog/unreleased/SOLR-18159-physical-memory-metrics.yml @@ -0,0 +1,9 @@ +title: Add new metric jvm_system_memory_bytes +type: added +authors: + - name: Jan Høydahl + url: https://home.apache.org/phonebook.html?uid=janhoy + - name: Matthew Biscocho +links: + - name: SOLR-18159 + url: https://issues.apache.org/jira/browse/SOLR-18159 diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeHealthApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeHealthApi.java new file mode 100644 index 000000000000..38ce0a20c9b8 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeHealthApi.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.endpoint; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.QueryParam; +import org.apache.solr.client.api.model.NodeHealthResponse; + +/** V2 API definition for checking the health of a Solr node. */ +@Path("/node/health") +public interface NodeHealthApi { + + @GET + @Operation( + summary = "Determine the health of a Solr node.", + tags = {"node"}) + NodeHealthResponse healthcheck( + @QueryParam("requireHealthyCores") Boolean requireHealthyCores, + @Parameter( + description = + "Maximum number of index generations a follower replica may lag behind its" + + " leader before the health check reports FAILURE. Only relevant when" + + " running in Standalone mode with leader/follower replication.") + @QueryParam("maxGenerationLag") + Integer maxGenerationLag); +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/NodeHealthResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/NodeHealthResponse.java new file mode 100644 index 000000000000..a0be8723b98a --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/NodeHealthResponse.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** Response body for the '/api/node/health' endpoint. */ +public class NodeHealthResponse extends SolrJerseyResponse { + + /** The possible health statuses for a Solr node. */ + public enum NodeStatus { + OK, + FAILURE + } + + @JsonProperty public NodeStatus status; + + @JsonProperty public String message; + + @JsonProperty("num_cores_unhealthy") + public Integer numCoresUnhealthy; +} diff --git a/solr/benchmark/src/java/org/apache/solr/bench/MiniClusterState.java b/solr/benchmark/src/java/org/apache/solr/bench/MiniClusterState.java deleted file mode 100755 index 4c4946a3ae86..000000000000 --- a/solr/benchmark/src/java/org/apache/solr/bench/MiniClusterState.java +++ /dev/null @@ -1,587 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.bench; - -import static org.apache.commons.io.file.PathUtils.deleteDirectory; -import static org.apache.solr.bench.BaseBenchState.log; - -import com.codahale.metrics.Meter; -import java.io.IOException; -import java.io.OutputStream; -import java.io.PrintStream; -import java.lang.management.ManagementFactory; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import java.util.SplittableRandom; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.jetty.HttpJettySolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.QueryRequest; -import org.apache.solr.client.solrj.request.SolrQuery; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.cloud.MiniSolrCloudCluster; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.IOUtils; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.SolrNamedThreadFactory; -import org.apache.solr.common.util.SuppressForbidden; -import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.util.SolrTestNonSecureRandomProvider; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.TearDown; -import org.openjdk.jmh.infra.BenchmarkParams; -import org.openjdk.jmh.infra.Control; - -/** The base class for Solr JMH benchmarks that operate against a {@code MiniSolrCloudCluster}. */ -public class MiniClusterState { - - /** The constant PROC_COUNT. */ - public static final int PROC_COUNT = - ManagementFactory.getOperatingSystemMXBean().getAvailableProcessors(); - - /** The type Mini cluster bench state. */ - @State(Scope.Benchmark) - public static class MiniClusterBenchState { - - /** The Metrics enabled. */ - boolean metricsEnabled = true; - - /** The Nodes. */ - public List nodes; - - public String zkHost; - - /** The Cluster. */ - MiniSolrCloudCluster cluster; - - /** The Client. */ - public HttpJettySolrClient client; - - /** The Run cnt. */ - int runCnt = 0; - - /** The Create collection and index. */ - boolean createCollectionAndIndex = true; - - /** The Delete mini cluster. */ - boolean deleteMiniCluster = true; - - /** Unless overridden we ensure SecureRandoms do not block. */ - boolean doNotWeakenSecureRandom = Boolean.getBoolean("doNotWeakenSecureRandom"); - - /** The Mini cluster base dir. */ - Path miniClusterBaseDir; - - /** To Allow cluster reuse. */ - boolean allowClusterReuse = false; - - /** The Is warmup. */ - boolean isWarmup; - - private SplittableRandom random; - private String workDir; - - private boolean useHttp1 = Boolean.getBoolean("solr.http1"); - - /** - * Tear down. - * - * @param benchmarkParams the benchmark params - * @throws Exception the exception - */ - @TearDown(Level.Iteration) - public void tearDown(BenchmarkParams benchmarkParams) throws Exception { - - // dump Solr metrics - Path metricsResults = - Path.of( - workDir, - "metrics-results", - benchmarkParams.id(), - String.valueOf(runCnt++), - benchmarkParams.getBenchmark() + ".txt"); - Files.createDirectories(metricsResults.getParent()); - - cluster.dumpMetrics(metricsResults.getParent(), metricsResults.getFileName().toString()); - } - - /** - * Check warm up. - * - * @param control the control - * @throws Exception the exception - */ - @Setup(Level.Iteration) - public void checkWarmUp(Control control) throws Exception { - isWarmup = control.stopMeasurement; - } - - /** - * Shutdown mini cluster. - * - * @param benchmarkParams the benchmark params - * @throws Exception the exception - */ - @TearDown(Level.Trial) - public void shutdownMiniCluster(BenchmarkParams benchmarkParams, BaseBenchState baseBenchState) - throws Exception { - BaseBenchState.dumpHeap(benchmarkParams); - - IOUtils.closeQuietly(client); - cluster.shutdown(); - logClusterDirectorySize(); - } - - private void logClusterDirectorySize() throws IOException { - log(""); - Files.list(miniClusterBaseDir.toAbsolutePath()) - .forEach( - (node) -> { - try { - long clusterSize = - Files.walk(node) - .filter(Files::isRegularFile) - .mapToLong( - file -> { - try { - return Files.size(file); - } catch (IOException e) { - throw new RuntimeException(e); - } - }) - .sum(); - log("mini cluster node size (bytes) " + node + " " + clusterSize); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - } - - /** - * Do setup. - * - * @param benchmarkParams the benchmark params - * @param baseBenchState the base bench state - * @throws Exception the exception - */ - @Setup(Level.Trial) - public void doSetup(BenchmarkParams benchmarkParams, BaseBenchState baseBenchState) - throws Exception { - - if (!doNotWeakenSecureRandom) { - // remove all blocking from all secure randoms - SolrTestNonSecureRandomProvider.injectProvider(); - } - - workDir = System.getProperty("workBaseDir", "build/work"); - - Path currentRelativePath = Path.of(""); - String s = currentRelativePath.toAbsolutePath().toString(); - log("current relative path is: " + s); - log("work path is: " + workDir); - - System.setProperty("doNotWaitForMergesOnIWClose", "true"); - - System.setProperty("pkiHandlerPrivateKeyPath", ""); - System.setProperty("pkiHandlerPublicKeyPath", ""); - - System.setProperty("solr.configset.default.confdir", "../server/solr/configsets/_default"); - - this.random = new SplittableRandom(BaseBenchState.getRandomSeed()); - - // not currently usable, but would enable JettySolrRunner's ill-conceived jetty.testMode and - // allow using SSL - - // System.getProperty("jetty.testMode", "true"); - // SolrCloudTestCase.sslConfig = SolrTestCaseJ4.buildSSLConfig(); - - String baseDirSysProp = System.getProperty("miniClusterBaseDir"); - if (baseDirSysProp != null) { - deleteMiniCluster = false; - miniClusterBaseDir = Path.of(baseDirSysProp); - if (Files.exists(miniClusterBaseDir)) { - createCollectionAndIndex = false; - allowClusterReuse = true; - } - } else { - miniClusterBaseDir = Path.of(workDir, "mini-cluster"); - } - - System.setProperty("metricsEnabled", String.valueOf(metricsEnabled)); - } - - /** - * Metrics enabled. - * - * @param metricsEnabled the metrics enabled - */ - public void metricsEnabled(boolean metricsEnabled) { - this.metricsEnabled = metricsEnabled; - } - - /** - * Start mini cluster. - * - * @param nodeCount the node count - */ - public void startMiniCluster(int nodeCount) { - log("starting mini cluster at base directory: " + miniClusterBaseDir.toAbsolutePath()); - - if (!allowClusterReuse && Files.exists(miniClusterBaseDir)) { - log("mini cluster base directory exists, removing ..."); - try { - deleteDirectory(miniClusterBaseDir); - } catch (IOException e) { - throw new RuntimeException(e); - } - createCollectionAndIndex = true; - } else if (Files.exists(miniClusterBaseDir)) { - createCollectionAndIndex = false; - deleteMiniCluster = false; - } - - try { - cluster = - new MiniSolrCloudCluster.Builder(nodeCount, miniClusterBaseDir) - .formatZkServer(false) - .addConfig("conf", getFile("src/resources/configs/cloud-minimal/conf")) - .configure(); - } catch (Exception e) { - if (Files.exists(miniClusterBaseDir)) { - try { - deleteDirectory(miniClusterBaseDir); - } catch (IOException ex) { - e.addSuppressed(ex); - } - } - throw new RuntimeException(e); - } - - nodes = new ArrayList<>(nodeCount); - List jetties = cluster.getJettySolrRunners(); - for (JettySolrRunner runner : jetties) { - nodes.add(runner.getBaseUrl().toString()); - } - zkHost = cluster.getZkServer().getZkAddress(); - - client = new HttpJettySolrClient.Builder(nodes.get(0)).useHttp1_1(useHttp1).build(); - - log("done starting mini cluster"); - log(""); - } - - /** - * Gets random. - * - * @return the random - */ - public SplittableRandom getRandom() { - return random; - } - - /** - * Create collection. - * - * @param collection the collection - * @param numShards the num shards - * @param numReplicas the num replicas - * @throws Exception the exception - */ - public void createCollection(String collection, int numShards, int numReplicas) - throws Exception { - if (createCollectionAndIndex) { - try { - - CollectionAdminRequest.Create request = - CollectionAdminRequest.createCollection(collection, "conf", numShards, numReplicas); - client.requestWithBaseUrl( - nodes.get(random.nextInt(cluster.getJettySolrRunners().size())), request, null); - - cluster.waitForActiveCollection( - collection, 15, TimeUnit.SECONDS, numShards, numShards * numReplicas); - } catch (Exception e) { - if (Files.exists(miniClusterBaseDir)) { - deleteDirectory(miniClusterBaseDir); - } - throw e; - } - } - } - - /** Setting useHttp1 to true will make the {@link #client} use http1 */ - public void setUseHttp1(boolean useHttp1) { - if (client != null) { - throw new IllegalStateException( - "You can only change this setting before starting the Mini Cluster"); - } - this.useHttp1 = useHttp1; - } - - @SuppressForbidden(reason = "This module does not need to deal with logging context") - public void index(String collection, Docs docs, int docCount) throws Exception { - index(collection, docs, docCount, true); - } - - /** - * Index. - * - * @param collection the collection - * @param docs the docs - * @param docCount the doc count - * @throws Exception the exception - */ - public void index(String collection, Docs docs, int docCount, boolean parallel) - throws Exception { - if (createCollectionAndIndex) { - log("indexing data for benchmark..."); - if (parallel) { - indexParallel(collection, docs, docCount); - } else { - indexBatch(collection, docs, docCount, 10000); - } - log("done indexing data for benchmark"); - - log("committing data ..."); - UpdateRequest commitRequest = new UpdateRequest(); - final var url = nodes.get(random.nextInt(cluster.getJettySolrRunners().size())); - commitRequest.setAction(UpdateRequest.ACTION.COMMIT, false, true); - client.requestWithBaseUrl(url, commitRequest, collection); - log("done committing data"); - } else { - cluster.waitForActiveCollection(collection, 15, TimeUnit.SECONDS); - } - - QueryRequest queryRequest = new QueryRequest(new SolrQuery("q", "*:*", "rows", "1")); - final var url = nodes.get(random.nextInt(cluster.getJettySolrRunners().size())); - NamedList result = client.requestWithBaseUrl(url, queryRequest, collection); - - log("sanity check of single row query result: " + result); - log(""); - - log("Dump Core Info"); - dumpCoreInfo(); - } - - @SuppressForbidden(reason = "This module does not need to deal with logging context") - private void indexParallel(String collection, Docs docs, int docCount) - throws InterruptedException { - Meter meter = new Meter(); - ExecutorService executorService = - Executors.newFixedThreadPool( - Runtime.getRuntime().availableProcessors(), - new SolrNamedThreadFactory("SolrJMH Indexer")); - ScheduledExecutorService scheduledExecutor = - Executors.newSingleThreadScheduledExecutor( - new SolrNamedThreadFactory("SolrJMH Indexer Progress")); - scheduledExecutor.scheduleAtFixedRate( - () -> { - if (meter.getCount() == docCount) { - scheduledExecutor.shutdown(); - } else { - log(meter.getCount() + " docs at " + meter.getMeanRate() + " doc/s"); - } - }, - 10, - 10, - TimeUnit.SECONDS); - for (int i = 0; i < docCount; i++) { - executorService.execute( - new Runnable() { - final SplittableRandom threadRandom = random.split(); - - @Override - public void run() { - UpdateRequest updateRequest = new UpdateRequest(); - final var url = - nodes.get(threadRandom.nextInt(cluster.getJettySolrRunners().size())); - SolrInputDocument doc = docs.inputDocument(); - // log("add doc " + doc); - updateRequest.add(doc); - meter.mark(); - - try { - client.requestWithBaseUrl(url, updateRequest, collection); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - }); - } - - log("done adding docs, waiting for executor to terminate..."); - - executorService.shutdown(); - boolean result = false; - while (!result) { - result = executorService.awaitTermination(600, TimeUnit.MINUTES); - } - - scheduledExecutor.shutdown(); - } - - private void indexBatch(String collection, Docs docs, int docCount, int batchSize) - throws SolrServerException, IOException { - Meter meter = new Meter(); - List batch = new ArrayList<>(batchSize); - for (int i = 1; i <= docCount; i++) { - batch.add(docs.inputDocument()); - if (i % batchSize == 0) { - UpdateRequest updateRequest = new UpdateRequest(); - updateRequest.add(batch); - client.requestWithBaseUrl(nodes.get(0), updateRequest, collection); - meter.mark(batch.size()); - batch.clear(); - log(meter.getCount() + " docs at " + (long) meter.getMeanRate() + " doc/s"); - } - } - if (!batch.isEmpty()) { - UpdateRequest updateRequest = new UpdateRequest(); - updateRequest.add(batch); - client.requestWithBaseUrl(nodes.get(0), updateRequest, collection); - meter.mark(batch.size()); - batch = null; - } - log(meter.getCount() + " docs at " + (long) meter.getMeanRate() + " doc/s"); - } - - /** - * Wait for merges. - * - * @param collection the collection - * @throws Exception the exception - */ - public void waitForMerges(String collection) throws Exception { - forceMerge(collection, Integer.MAX_VALUE); - } - - /** - * Force merge. - * - * @param collection the collection - * @param maxMergeSegments the max merge segments - * @throws Exception the exception - */ - public void forceMerge(String collection, int maxMergeSegments) throws Exception { - if (createCollectionAndIndex) { - // we control segment count for a more informative benchmark *and* because background - // merging would continue after - // indexing and overlap with the benchmark - if (maxMergeSegments == Integer.MAX_VALUE) { - log("waiting for merges to finish...\n"); - } else { - log("merging segments to " + maxMergeSegments + " segments ...\n"); - } - - UpdateRequest optimizeRequest = new UpdateRequest(); - final var url = nodes.get(random.nextInt(cluster.getJettySolrRunners().size())); - optimizeRequest.setAction(UpdateRequest.ACTION.OPTIMIZE, false, true, maxMergeSegments); - client.requestWithBaseUrl(url, optimizeRequest, collection); - } - } - - /** - * Dump core info. - * - * @throws IOException the io exception - */ - @SuppressForbidden(reason = "JMH uses std out for user output") - public void dumpCoreInfo() throws IOException { - cluster.dumpCoreInfo( - !BaseBenchState.QUIET_LOG - ? System.out - : new PrintStream(OutputStream.nullOutputStream(), false, StandardCharsets.UTF_8)); - } - } - - /** - * Params modifiable solr params. - * - * @param moreParams the more params - * @return the modifiable solr params - */ - public static ModifiableSolrParams params(String... moreParams) { - ModifiableSolrParams params = new ModifiableSolrParams(); - for (int i = 0; i < moreParams.length; i += 2) { - params.add(moreParams[i], moreParams[i + 1]); - } - return params; - } - - /** - * Params modifiable solr params. - * - * @param params the params - * @param moreParams the more params - * @return the modifiable solr params - */ - public static ModifiableSolrParams params(ModifiableSolrParams params, String... moreParams) { - for (int i = 0; i < moreParams.length; i += 2) { - params.add(moreParams[i], moreParams[i + 1]); - } - return params; - } - - /** - * Gets file. - * - * @param name the name - * @return the file - */ - public static Path getFile(String name) { - final URL url = - MiniClusterState.class - .getClassLoader() - .getResource(name.replace(FileSystems.getDefault().getSeparator(), "/")); - if (url != null) { - try { - return Path.of(url.toURI()); - } catch (Exception e) { - throw new RuntimeException( - "Resource was found on classpath, but cannot be resolved to a " - + "normal file (maybe it is part of a JAR file): " - + name); - } - } - Path file = Path.of(name); - if (Files.exists(file)) { - return file; - } else { - file = Path.of("../../../", name); - if (Files.exists(file)) { - return file; - } - } - throw new RuntimeException( - "Cannot find resource in classpath or in file-system (relative to CWD): " - + name - + " CWD=" - + Path.of("").toAbsolutePath()); - } -} diff --git a/solr/benchmark/src/java/org/apache/solr/bench/SolrBenchState.java b/solr/benchmark/src/java/org/apache/solr/bench/SolrBenchState.java new file mode 100755 index 000000000000..1aade6389ddc --- /dev/null +++ b/solr/benchmark/src/java/org/apache/solr/bench/SolrBenchState.java @@ -0,0 +1,580 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.bench; + +import static org.apache.commons.io.file.PathUtils.deleteDirectory; +import static org.apache.solr.bench.BaseBenchState.log; + +import com.codahale.metrics.Meter; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.lang.management.ManagementFactory; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.SplittableRandom; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.jetty.HttpJettySolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.client.solrj.request.SolrQuery; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.cloud.MiniSolrCloudCluster; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.IOUtils; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SolrNamedThreadFactory; +import org.apache.solr.common.util.SuppressForbidden; +import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.util.SolrTestNonSecureRandomProvider; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.Control; + +/** JMH {@link State} class for Solr benchmarks. This is the benchmark's facade to Solr. */ +@State(Scope.Benchmark) +public class SolrBenchState { + + /** The constant PROC_COUNT. */ + public static final int PROC_COUNT = + ManagementFactory.getOperatingSystemMXBean().getAvailableProcessors(); + + /** The Metrics enabled. */ + boolean metricsEnabled = true; + + /** The Nodes. */ + public List nodes; + + public String zkHost; + + /** The Cluster. */ + MiniSolrCloudCluster cluster; + + /** The Client. */ + public HttpJettySolrClient client; + + /** The Run cnt. */ + int runCnt = 0; + + /** The Create collection and index. */ + boolean createCollectionAndIndex = true; + + /** The Delete mini cluster. */ + boolean deleteMiniCluster = true; + + /** Unless overridden we ensure SecureRandoms do not block. */ + boolean doNotWeakenSecureRandom = Boolean.getBoolean("doNotWeakenSecureRandom"); + + /** The Mini cluster base dir. */ + Path miniClusterBaseDir; + + /** To Allow cluster reuse. */ + boolean allowClusterReuse = false; + + /** The Is warmup. */ + boolean isWarmup; + + private SplittableRandom random; + private String workDir; + + private boolean useHttp1 = Boolean.getBoolean("solr.http1"); + + /** + * Tear down. + * + * @param benchmarkParams the benchmark params + * @throws Exception the exception + */ + @TearDown(Level.Iteration) + public void tearDown(BenchmarkParams benchmarkParams) throws Exception { + + // dump Solr metrics + Path metricsResults = + Path.of( + workDir, + "metrics-results", + benchmarkParams.id(), + String.valueOf(runCnt++), + benchmarkParams.getBenchmark() + ".txt"); + Files.createDirectories(metricsResults.getParent()); + + cluster.dumpMetrics(metricsResults.getParent(), metricsResults.getFileName().toString()); + } + + /** + * Check warm up. + * + * @param control the control + * @throws Exception the exception + */ + @Setup(Level.Iteration) + public void checkWarmUp(Control control) throws Exception { + isWarmup = control.stopMeasurement; + } + + /** + * Shutdown Solr. + * + * @param benchmarkParams the benchmark params + * @throws Exception the exception + */ + @TearDown(Level.Trial) + public void shutdownSolr(BenchmarkParams benchmarkParams, BaseBenchState baseBenchState) + throws Exception { + BaseBenchState.dumpHeap(benchmarkParams); + + IOUtils.closeQuietly(client); + cluster.shutdown(); + logClusterDirectorySize(); + } + + private void logClusterDirectorySize() throws IOException { + log(""); + Files.list(miniClusterBaseDir.toAbsolutePath()) + .forEach( + (node) -> { + try { + long clusterSize = + Files.walk(node) + .filter(Files::isRegularFile) + .mapToLong( + file -> { + try { + return Files.size(file); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .sum(); + log("mini cluster node size (bytes) " + node + " " + clusterSize); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + /** + * Do setup. + * + * @param benchmarkParams the benchmark params + * @param baseBenchState the base bench state + * @throws Exception the exception + */ + @Setup(Level.Trial) + public void doSetup(BenchmarkParams benchmarkParams, BaseBenchState baseBenchState) + throws Exception { + + if (!doNotWeakenSecureRandom) { + // remove all blocking from all secure randoms + SolrTestNonSecureRandomProvider.injectProvider(); + } + + workDir = System.getProperty("workBaseDir", "build/work"); + + Path currentRelativePath = Path.of(""); + String s = currentRelativePath.toAbsolutePath().toString(); + log("current relative path is: " + s); + log("work path is: " + workDir); + + System.setProperty("doNotWaitForMergesOnIWClose", "true"); + + System.setProperty("pkiHandlerPrivateKeyPath", ""); + System.setProperty("pkiHandlerPublicKeyPath", ""); + + System.setProperty("solr.configset.default.confdir", "../server/solr/configsets/_default"); + + this.random = new SplittableRandom(BaseBenchState.getRandomSeed()); + + // not currently usable, but would enable JettySolrRunner's ill-conceived jetty.testMode and + // allow using SSL + + // System.getProperty("jetty.testMode", "true"); + // SolrCloudTestCase.sslConfig = SolrTestCaseJ4.buildSSLConfig(); + + String baseDirSysProp = System.getProperty("miniClusterBaseDir"); + if (baseDirSysProp != null) { + deleteMiniCluster = false; + miniClusterBaseDir = Path.of(baseDirSysProp); + if (Files.exists(miniClusterBaseDir)) { + createCollectionAndIndex = false; + allowClusterReuse = true; + } + } else { + miniClusterBaseDir = Path.of(workDir, "mini-cluster"); + } + + System.setProperty("metricsEnabled", String.valueOf(metricsEnabled)); + } + + /** + * Metrics enabled. + * + * @param metricsEnabled the metrics enabled + */ + public void metricsEnabled(boolean metricsEnabled) { + this.metricsEnabled = metricsEnabled; + } + + /** + * Start Solr. + * + * @param nodeCount the node count + */ + public void startSolr(int nodeCount) { + log("starting mini cluster at base directory: " + miniClusterBaseDir.toAbsolutePath()); + + if (!allowClusterReuse && Files.exists(miniClusterBaseDir)) { + log("mini cluster base directory exists, removing ..."); + try { + deleteDirectory(miniClusterBaseDir); + } catch (IOException e) { + throw new RuntimeException(e); + } + createCollectionAndIndex = true; + } else if (Files.exists(miniClusterBaseDir)) { + createCollectionAndIndex = false; + deleteMiniCluster = false; + } + + try { + cluster = + new MiniSolrCloudCluster.Builder(nodeCount, miniClusterBaseDir) + .formatZkServer(false) + .addConfig("conf", getFile("src/resources/configs/cloud-minimal/conf")) + .configure(); + } catch (Exception e) { + if (Files.exists(miniClusterBaseDir)) { + try { + deleteDirectory(miniClusterBaseDir); + } catch (IOException ex) { + e.addSuppressed(ex); + } + } + throw new RuntimeException(e); + } + + nodes = new ArrayList<>(nodeCount); + List jetties = cluster.getJettySolrRunners(); + for (JettySolrRunner runner : jetties) { + nodes.add(runner.getBaseUrl().toString()); + } + zkHost = cluster.getZkServer().getZkAddress(); + + client = new HttpJettySolrClient.Builder(nodes.get(0)).useHttp1_1(useHttp1).build(); + + log("done starting mini cluster"); + log(""); + } + + /** + * Gets random. + * + * @return the random + */ + public SplittableRandom getRandom() { + return random; + } + + /** + * Create collection. + * + * @param collection the collection + * @param numShards the num shards + * @param numReplicas the num replicas + * @throws Exception the exception + */ + public void createCollection(String collection, int numShards, int numReplicas) throws Exception { + if (createCollectionAndIndex) { + try { + + CollectionAdminRequest.Create request = + CollectionAdminRequest.createCollection(collection, "conf", numShards, numReplicas); + client.requestWithBaseUrl( + nodes.get(random.nextInt(cluster.getJettySolrRunners().size())), request, null); + + cluster.waitForActiveCollection( + collection, 15, TimeUnit.SECONDS, numShards, numShards * numReplicas); + } catch (Exception e) { + if (Files.exists(miniClusterBaseDir)) { + deleteDirectory(miniClusterBaseDir); + } + throw e; + } + } + } + + /** Setting useHttp1 to true will make the {@link #client} use http1 */ + public void setUseHttp1(boolean useHttp1) { + if (client != null) { + throw new IllegalStateException( + "You can only change this setting before starting the Mini Cluster"); + } + this.useHttp1 = useHttp1; + } + + @SuppressForbidden(reason = "This module does not need to deal with logging context") + public void index(String collection, Docs docs, int docCount) throws Exception { + index(collection, docs, docCount, true); + } + + /** + * Index. + * + * @param collection the collection + * @param docs the docs + * @param docCount the doc count + * @throws Exception the exception + */ + public void index(String collection, Docs docs, int docCount, boolean parallel) throws Exception { + if (createCollectionAndIndex) { + log("indexing data for benchmark..."); + if (parallel) { + indexParallel(collection, docs, docCount); + } else { + indexBatch(collection, docs, docCount, 10000); + } + log("done indexing data for benchmark"); + + log("committing data ..."); + UpdateRequest commitRequest = new UpdateRequest(); + final var url = nodes.get(random.nextInt(cluster.getJettySolrRunners().size())); + commitRequest.setAction(UpdateRequest.ACTION.COMMIT, false, true); + client.requestWithBaseUrl(url, commitRequest, collection); + log("done committing data"); + } else { + cluster.waitForActiveCollection(collection, 15, TimeUnit.SECONDS); + } + + QueryRequest queryRequest = new QueryRequest(new SolrQuery("q", "*:*", "rows", "1")); + final var url = nodes.get(random.nextInt(cluster.getJettySolrRunners().size())); + NamedList result = client.requestWithBaseUrl(url, queryRequest, collection); + + log("sanity check of single row query result: " + result); + log(""); + + log("Dump Core Info"); + dumpCoreInfo(); + } + + @SuppressForbidden(reason = "This module does not need to deal with logging context") + private void indexParallel(String collection, Docs docs, int docCount) + throws InterruptedException { + Meter meter = new Meter(); + ExecutorService executorService = + Executors.newFixedThreadPool( + Runtime.getRuntime().availableProcessors(), + new SolrNamedThreadFactory("SolrJMH Indexer")); + ScheduledExecutorService scheduledExecutor = + Executors.newSingleThreadScheduledExecutor( + new SolrNamedThreadFactory("SolrJMH Indexer Progress")); + scheduledExecutor.scheduleAtFixedRate( + () -> { + if (meter.getCount() == docCount) { + scheduledExecutor.shutdown(); + } else { + log(meter.getCount() + " docs at " + meter.getMeanRate() + " doc/s"); + } + }, + 10, + 10, + TimeUnit.SECONDS); + for (int i = 0; i < docCount; i++) { + executorService.execute( + new Runnable() { + final SplittableRandom threadRandom = random.split(); + + @Override + public void run() { + UpdateRequest updateRequest = new UpdateRequest(); + final var url = nodes.get(threadRandom.nextInt(cluster.getJettySolrRunners().size())); + SolrInputDocument doc = docs.inputDocument(); + // log("add doc " + doc); + updateRequest.add(doc); + meter.mark(); + + try { + client.requestWithBaseUrl(url, updateRequest, collection); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); + } + + log("done adding docs, waiting for executor to terminate..."); + + executorService.shutdown(); + boolean result = false; + while (!result) { + result = executorService.awaitTermination(600, TimeUnit.MINUTES); + } + + scheduledExecutor.shutdown(); + } + + private void indexBatch(String collection, Docs docs, int docCount, int batchSize) + throws SolrServerException, IOException { + Meter meter = new Meter(); + List batch = new ArrayList<>(batchSize); + for (int i = 1; i <= docCount; i++) { + batch.add(docs.inputDocument()); + if (i % batchSize == 0) { + UpdateRequest updateRequest = new UpdateRequest(); + updateRequest.add(batch); + client.requestWithBaseUrl(nodes.get(0), updateRequest, collection); + meter.mark(batch.size()); + batch.clear(); + log(meter.getCount() + " docs at " + (long) meter.getMeanRate() + " doc/s"); + } + } + if (!batch.isEmpty()) { + UpdateRequest updateRequest = new UpdateRequest(); + updateRequest.add(batch); + client.requestWithBaseUrl(nodes.get(0), updateRequest, collection); + meter.mark(batch.size()); + batch = null; + } + log(meter.getCount() + " docs at " + (long) meter.getMeanRate() + " doc/s"); + } + + /** + * Wait for merges. + * + * @param collection the collection + * @throws Exception the exception + */ + public void waitForMerges(String collection) throws Exception { + forceMerge(collection, Integer.MAX_VALUE); + } + + /** + * Force merge. + * + * @param collection the collection + * @param maxMergeSegments the max merge segments + * @throws Exception the exception + */ + public void forceMerge(String collection, int maxMergeSegments) throws Exception { + if (createCollectionAndIndex) { + // we control segment count for a more informative benchmark *and* because background + // merging would continue after + // indexing and overlap with the benchmark + if (maxMergeSegments == Integer.MAX_VALUE) { + log("waiting for merges to finish...\n"); + } else { + log("merging segments to " + maxMergeSegments + " segments ...\n"); + } + + UpdateRequest optimizeRequest = new UpdateRequest(); + final var url = nodes.get(random.nextInt(cluster.getJettySolrRunners().size())); + optimizeRequest.setAction(UpdateRequest.ACTION.OPTIMIZE, false, true, maxMergeSegments); + client.requestWithBaseUrl(url, optimizeRequest, collection); + } + } + + /** + * Dump core info. + * + * @throws IOException the io exception + */ + @SuppressForbidden(reason = "JMH uses std out for user output") + public void dumpCoreInfo() throws IOException { + cluster.dumpCoreInfo( + !BaseBenchState.QUIET_LOG + ? System.out + : new PrintStream(OutputStream.nullOutputStream(), false, StandardCharsets.UTF_8)); + } + + /** + * Params modifiable solr params. + * + * @param moreParams the more params + * @return the modifiable solr params + */ + public static ModifiableSolrParams params(String... moreParams) { + ModifiableSolrParams params = new ModifiableSolrParams(); + for (int i = 0; i < moreParams.length; i += 2) { + params.add(moreParams[i], moreParams[i + 1]); + } + return params; + } + + /** + * Params modifiable solr params. + * + * @param params the params + * @param moreParams the more params + * @return the modifiable solr params + */ + public static ModifiableSolrParams params(ModifiableSolrParams params, String... moreParams) { + for (int i = 0; i < moreParams.length; i += 2) { + params.add(moreParams[i], moreParams[i + 1]); + } + return params; + } + + /** + * Gets file. + * + * @param name the name + * @return the file + */ + public static Path getFile(String name) { + final URL url = + SolrBenchState.class + .getClassLoader() + .getResource(name.replace(FileSystems.getDefault().getSeparator(), "/")); + if (url != null) { + try { + return Path.of(url.toURI()); + } catch (Exception e) { + throw new RuntimeException( + "Resource was found on classpath, but cannot be resolved to a " + + "normal file (maybe it is part of a JAR file): " + + name); + } + } + Path file = Path.of(name); + if (Files.exists(file)) { + return file; + } else { + file = Path.of("../../../", name); + if (Files.exists(file)) { + return file; + } + } + throw new RuntimeException( + "Cannot find resource in classpath or in file-system (relative to CWD): " + + name + + " CWD=" + + Path.of("").toAbsolutePath()); + } +} diff --git a/solr/benchmark/src/java/org/apache/solr/bench/index/CloudIndexing.java b/solr/benchmark/src/java/org/apache/solr/bench/index/CloudIndexing.java index 5f6c6526690d..770699d2a54e 100755 --- a/solr/benchmark/src/java/org/apache/solr/bench/index/CloudIndexing.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/index/CloudIndexing.java @@ -24,7 +24,7 @@ import java.util.Iterator; import java.util.concurrent.TimeUnit; import org.apache.solr.bench.Docs; -import org.apache.solr.bench.MiniClusterState; +import org.apache.solr.bench.SolrBenchState; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.common.SolrInputDocument; import org.openjdk.jmh.annotations.Benchmark; @@ -114,22 +114,20 @@ private void preGenerate() { } @Setup(Level.Trial) - public void doSetup(MiniClusterState.MiniClusterBenchState miniClusterState) throws Exception { + public void doSetup(SolrBenchState solrBenchState) throws Exception { preGenerate(); System.setProperty("mergePolicyFactory", "org.apache.solr.index.NoMergePolicyFactory"); - miniClusterState.startMiniCluster(nodeCount); - miniClusterState.createCollection(COLLECTION, numShards, numReplicas); + solrBenchState.startSolr(nodeCount); + solrBenchState.createCollection(COLLECTION, numShards, numReplicas); } } @Benchmark - public Object indexDoc(MiniClusterState.MiniClusterBenchState miniClusterState, BenchState state) - throws Exception { + public Object indexDoc(SolrBenchState solrBenchState, BenchState state) throws Exception { UpdateRequest updateRequest = new UpdateRequest(); updateRequest.add(state.getNextDoc()); - final var url = - miniClusterState.nodes.get(miniClusterState.getRandom().nextInt(state.nodeCount)); - return miniClusterState.client.requestWithBaseUrl(url, updateRequest, BenchState.COLLECTION); + final var url = solrBenchState.nodes.get(solrBenchState.getRandom().nextInt(state.nodeCount)); + return solrBenchState.client.requestWithBaseUrl(url, updateRequest, BenchState.COLLECTION); } } diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/ExitableDirectoryReaderSearch.java b/solr/benchmark/src/java/org/apache/solr/bench/search/ExitableDirectoryReaderSearch.java index 0b7b4a5ff5a6..0365fe004105 100644 --- a/solr/benchmark/src/java/org/apache/solr/bench/search/ExitableDirectoryReaderSearch.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/search/ExitableDirectoryReaderSearch.java @@ -24,7 +24,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import org.apache.solr.bench.Docs; -import org.apache.solr.bench.MiniClusterState; +import org.apache.solr.bench.SolrBenchState; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.response.QueryResponse; @@ -67,9 +67,8 @@ public static class BenchState { int WORDS = NUM_DOCS / 100; @Setup(Level.Trial) - public void setupTrial(MiniClusterState.MiniClusterBenchState miniClusterState) - throws Exception { - miniClusterState.setUseHttp1(true); + public void setupTrial(SolrBenchState solrBenchState) throws Exception { + solrBenchState.setUseHttp1(true); System.setProperty("documentCache.enabled", "false"); System.setProperty("queryResultCache.enabled", "false"); System.setProperty("filterCache.enabled", "false"); @@ -78,9 +77,9 @@ public void setupTrial(MiniClusterState.MiniClusterBenchState miniClusterState) System.setProperty("segmentsPerTier", "200"); System.setProperty("maxBufferedDocs", "100"); - miniClusterState.startMiniCluster(1); + solrBenchState.startSolr(1); log("######### Creating index ..."); - miniClusterState.createCollection(COLLECTION, 1, 1); + solrBenchState.createCollection(COLLECTION, 1, 1); // create a lot of large-ish fields to scan positions Docs docs = Docs.docs(1234567890L) @@ -103,9 +102,9 @@ public void setupTrial(MiniClusterState.MiniClusterBenchState miniClusterState) .field( "f9_ts", strings().alpha().maxCardinality(WORDS).multi(50).ofLengthBetween(3, 10)); - miniClusterState.index(COLLECTION, docs, NUM_DOCS, true); - miniClusterState.forceMerge(COLLECTION, 200); - miniClusterState.dumpCoreInfo(); + solrBenchState.index(COLLECTION, docs, NUM_DOCS, true); + solrBenchState.forceMerge(COLLECTION, 200); + solrBenchState.dumpCoreInfo(); } // this adds significant processing time to the checking of query limits @@ -116,13 +115,13 @@ public void setupTrial(MiniClusterState.MiniClusterBenchState miniClusterState) private static final String matchExpression = "ExitableTermsEnum:-1"; @Setup(Level.Iteration) - public void setupQueries(MiniClusterState.MiniClusterBenchState state) throws Exception { + public void setupQueries(SolrBenchState solrBenchState) throws Exception { if (verifyEDRInUse) { TestInjection.queryTimeout = new CallerSpecificQueryLimit(Set.of(matchExpression)); } // reload collection to force searcher / reader refresh CollectionAdminRequest.Reload reload = CollectionAdminRequest.reloadCollection(COLLECTION); - state.client.request(reload); + solrBenchState.client.request(reload); queryFields = Docs.docs(1234567890L) @@ -154,25 +153,23 @@ public void tearDownTrial() throws Exception { private static ModifiableSolrParams createInitialParams() { ModifiableSolrParams params = - MiniClusterState.params("rows", "100", "timeAllowed", "1000", "fl", "*"); + SolrBenchState.params("rows", "100", "timeAllowed", "1000", "fl", "*"); return params; } @Benchmark - public void testShortQuery( - MiniClusterState.MiniClusterBenchState miniClusterState, Blackhole bh, BenchState state) + public void testShortQuery(SolrBenchState solrBenchState, Blackhole bh, BenchState state) throws Exception { SolrInputDocument queryDoc = state.queryFields.inputDocument(); ModifiableSolrParams params = createInitialParams(); params.set("q", "f1_ts:" + queryDoc.getFieldValue("f1_ts").toString()); QueryRequest queryRequest = new QueryRequest(params); - QueryResponse rsp = queryRequest.process(miniClusterState.client, COLLECTION); + QueryResponse rsp = queryRequest.process(solrBenchState.client, COLLECTION); bh.consume(rsp); } @Benchmark - public void testLongQuery( - MiniClusterState.MiniClusterBenchState miniClusterState, Blackhole bh, BenchState state) + public void testLongQuery(SolrBenchState solrBenchState, Blackhole bh, BenchState state) throws Exception { SolrInputDocument queryDoc = state.queryFields.inputDocument(); ModifiableSolrParams params = createInitialParams(); @@ -186,7 +183,7 @@ public void testLongQuery( } params.set("q", query.toString()); QueryRequest queryRequest = new QueryRequest(params); - QueryResponse rsp = queryRequest.process(miniClusterState.client, COLLECTION); + QueryResponse rsp = queryRequest.process(solrBenchState.client, COLLECTION); bh.consume(rsp); } } diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/FilterCache.java b/solr/benchmark/src/java/org/apache/solr/bench/search/FilterCache.java index 197a346bb3df..f736aa4bf802 100644 --- a/solr/benchmark/src/java/org/apache/solr/bench/search/FilterCache.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/search/FilterCache.java @@ -24,7 +24,7 @@ import java.nio.charset.StandardCharsets; import org.apache.solr.bench.BaseBenchState; import org.apache.solr.bench.Docs; -import org.apache.solr.bench.MiniClusterState; +import org.apache.solr.bench.SolrBenchState; import org.apache.solr.bench.SolrRandomnessSource; import org.apache.solr.bench.generators.SolrGen; import org.apache.solr.client.solrj.SolrServerException; @@ -71,8 +71,7 @@ public static class BenchState { String baseUrl; @Setup(Level.Trial) - public void setupTrial(MiniClusterState.MiniClusterBenchState miniClusterState) - throws Exception { + public void setupTrial(SolrBenchState solrBenchState) throws Exception { String cacheEnabled = cacheEnabledAsyncSize.split(":")[0]; String asyncCache = cacheEnabledAsyncSize.split(":")[1]; String cacheSize = cacheEnabledAsyncSize.split(":")[2]; @@ -81,8 +80,8 @@ public void setupTrial(MiniClusterState.MiniClusterBenchState miniClusterState) System.setProperty("filterCache.initialSize", cacheSize); System.setProperty("filterCache.async", asyncCache); - miniClusterState.startMiniCluster(1); - miniClusterState.createCollection(COLLECTION, 1, 1); + solrBenchState.startSolr(1); + solrBenchState.createCollection(COLLECTION, 1, 1); Docs docs = Docs.docs().field("id", integers().incrementing()); @@ -100,24 +99,24 @@ public Boolean generate(SolrRandomnessSource in) { docs.field("Ea_b", booleans); docs.field("FB_b", booleans); - miniClusterState.index(COLLECTION, docs, 30 * 1000); - baseUrl = miniClusterState.nodes.get(0); + solrBenchState.index(COLLECTION, docs, 30 * 1000); + baseUrl = solrBenchState.nodes.get(0); } @Setup(Level.Iteration) - public void setupIteration(MiniClusterState.MiniClusterBenchState miniClusterState) + public void setupIteration(SolrBenchState solrBenchState) throws SolrServerException, IOException { // Reload the collection/core to drop existing caches CollectionAdminRequest.Reload reload = CollectionAdminRequest.reloadCollection(COLLECTION); - miniClusterState.client.requestWithBaseUrl(miniClusterState.nodes.get(0), reload, null); + solrBenchState.client.requestWithBaseUrl(solrBenchState.nodes.get(0), reload, null); } @TearDown(Level.Iteration) - public void dumpMetrics(MiniClusterState.MiniClusterBenchState miniClusterState) { + public void dumpMetrics(SolrBenchState solrBenchState) { // TODO add a verbose flag String url = - miniClusterState.nodes.get(0) + solrBenchState.nodes.get(0) + "/admin/metrics?prefix=CACHE.searcher.filterCache&omitHeader=true"; HttpURLConnection conn = null; try { @@ -134,20 +133,17 @@ public void dumpMetrics(MiniClusterState.MiniClusterBenchState miniClusterState) } @Benchmark - public Object filterCacheMultipleQueries( - BenchState benchState, MiniClusterState.MiniClusterBenchState miniClusterState) + public Object filterCacheMultipleQueries(BenchState benchState, SolrBenchState solrBenchState) throws SolrServerException, IOException { - return miniClusterState.client.requestWithBaseUrl( + return solrBenchState.client.requestWithBaseUrl( benchState.baseUrl, - miniClusterState.getRandom().nextBoolean() ? benchState.q1 : benchState.q2, + solrBenchState.getRandom().nextBoolean() ? benchState.q1 : benchState.q2, COLLECTION); } @Benchmark - public Object filterCacheSingleQuery( - BenchState benchState, MiniClusterState.MiniClusterBenchState miniClusterState) + public Object filterCacheSingleQuery(BenchState benchState, SolrBenchState solrBenchState) throws SolrServerException, IOException { - return miniClusterState.client.requestWithBaseUrl( - benchState.baseUrl, benchState.q1, COLLECTION); + return solrBenchState.client.requestWithBaseUrl(benchState.baseUrl, benchState.q1, COLLECTION); } } diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/JsonFaceting.java b/solr/benchmark/src/java/org/apache/solr/bench/search/JsonFaceting.java index 9482f8a588b2..7c1216b41b59 100755 --- a/solr/benchmark/src/java/org/apache/solr/bench/search/JsonFaceting.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/search/JsonFaceting.java @@ -24,7 +24,7 @@ import java.util.concurrent.TimeUnit; import org.apache.solr.bench.BaseBenchState; import org.apache.solr.bench.Docs; -import org.apache.solr.bench.MiniClusterState; +import org.apache.solr.bench.SolrBenchState; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; @@ -99,16 +99,15 @@ public static class BenchState { private ModifiableSolrParams params; @Setup(Level.Trial) - public void setup( - BenchmarkParams benchmarkParams, MiniClusterState.MiniClusterBenchState miniClusterState) + public void setup(BenchmarkParams benchmarkParams, SolrBenchState solrBenchState) throws Exception { System.setProperty("maxMergeAtOnce", "50"); System.setProperty("segmentsPerTier", "50"); - miniClusterState.startMiniCluster(nodeCount); + solrBenchState.startSolr(nodeCount); - miniClusterState.createCollection(collection, numShards, numReplicas); + solrBenchState.createCollection(collection, numShards, numReplicas); // Define random documents Docs docs = @@ -132,12 +131,12 @@ public void setup( .field(integers().allWithMaxCardinality(facetCard2)) .field(integers().allWithMaxCardinality(facetCard2)); - miniClusterState.index(collection, docs, docCount); - miniClusterState.forceMerge(collection, 25); + solrBenchState.index(collection, docs, docCount); + solrBenchState.forceMerge(collection, 25); params = new ModifiableSolrParams(); - MiniClusterState.params( + SolrBenchState.params( params, "q", "*:*", @@ -167,7 +166,7 @@ public void setup( params.set("timeAllowed", "5000"); } - // MiniClusterState.log("params: " + params + "\n"); + // SolrBenchState.log("params: " + params + "\n"); } @State(Scope.Thread) @@ -185,17 +184,17 @@ public void setup() { @Benchmark @Timeout(time = 500, timeUnit = TimeUnit.SECONDS) public void jsonFacet( - MiniClusterState.MiniClusterBenchState miniClusterState, + SolrBenchState solrBenchState, BenchState state, BenchState.ThreadState threadState, Blackhole bh) throws Exception { - final var url = miniClusterState.nodes.get(threadState.random.nextInt(state.nodeCount)); + final var url = solrBenchState.nodes.get(threadState.random.nextInt(state.nodeCount)); QueryRequest queryRequest = new QueryRequest(state.params); NamedList result = - miniClusterState.client.requestWithBaseUrl(url, queryRequest, state.collection); + solrBenchState.client.requestWithBaseUrl(url, queryRequest, state.collection); - // MiniClusterState.log("result: " + result); + // SolrBenchState.log("result: " + result); bh.consume(result); } diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/NumericSearch.java b/solr/benchmark/src/java/org/apache/solr/bench/search/NumericSearch.java index e7f574cd341e..70d307963a6b 100644 --- a/solr/benchmark/src/java/org/apache/solr/bench/search/NumericSearch.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/search/NumericSearch.java @@ -27,7 +27,7 @@ import java.util.stream.Collectors; import org.apache.solr.bench.CircularIterator; import org.apache.solr.bench.Docs; -import org.apache.solr.bench.MiniClusterState; +import org.apache.solr.bench.SolrBenchState; import org.apache.solr.bench.generators.SolrGen; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.CollectionAdminRequest; @@ -68,11 +68,10 @@ public static class BenchState { Iterator queries; @Setup(Level.Trial) - public void setupTrial(MiniClusterState.MiniClusterBenchState miniClusterState) - throws Exception { - miniClusterState.setUseHttp1(true); - miniClusterState.startMiniCluster(1); - miniClusterState.createCollection(COLLECTION, 1, 1); + public void setupTrial(SolrBenchState solrBenchState) throws Exception { + solrBenchState.setUseHttp1(true); + solrBenchState.startSolr(1); + solrBenchState.createCollection(COLLECTION, 1, 1); int maxCardinality = 10000; int numDocs = 2000000; setValues = integers().allWithMaxCardinality(maxCardinality); @@ -93,16 +92,15 @@ public void setupTrial(MiniClusterState.MiniClusterBenchState miniClusterState) .field("term_high_s", highCardinalityTerms); // .field("numbers_dt", setValues); - miniClusterState.index(COLLECTION, docs, numDocs, false); - basePath = miniClusterState.nodes.get(0); + solrBenchState.index(COLLECTION, docs, numDocs, false); + basePath = solrBenchState.nodes.get(0); SolrQuery q = new SolrQuery("*:*"); q.setParam("facet", "true"); q.setParam("rows", "0"); q.setParam("facet.field", "numbers_i_dv", "term_low_s", "term_high_s"); q.setParam("facet.limit", String.valueOf(maxCardinality)); QueryRequest req = new QueryRequest(q); - QueryResponse response = - req.processWithBaseUrl(miniClusterState.client, basePath, COLLECTION); + QueryResponse response = req.processWithBaseUrl(solrBenchState.client, basePath, COLLECTION); Set numbers = response.getFacetField("numbers_i_dv").getValues().stream() .map(FacetField.Count::getName) @@ -140,11 +138,11 @@ private Set buildSetQueries(Set numbers) { } @Setup(Level.Iteration) - public void setupIteration(MiniClusterState.MiniClusterBenchState miniClusterState) + public void setupIteration(SolrBenchState solrBenchState) throws SolrServerException, IOException { // Reload the collection/core to drop existing caches CollectionAdminRequest.Reload reload = CollectionAdminRequest.reloadCollection(COLLECTION); - miniClusterState.client.requestWithBaseUrl(miniClusterState.nodes.get(0), reload, null); + solrBenchState.client.requestWithBaseUrl(solrBenchState.nodes.get(0), reload, null); } public QueryRequest intSetQuery(boolean dvs) { @@ -176,97 +174,75 @@ QueryRequest setQuery(String field) { } @Benchmark - public Object intSet( - Blackhole blackhole, - BenchState benchState, - MiniClusterState.MiniClusterBenchState miniClusterState) + public Object intSet(Blackhole blackhole, BenchState benchState, SolrBenchState solrBenchState) throws SolrServerException, IOException { QueryResponse response = - benchState.intSetQuery(false).process(miniClusterState.client, COLLECTION); + benchState.intSetQuery(false).process(solrBenchState.client, COLLECTION); blackhole.consume(response); return response; } @Benchmark - public Object longSet( - Blackhole blackhole, - BenchState benchState, - MiniClusterState.MiniClusterBenchState miniClusterState) + public Object longSet(Blackhole blackhole, BenchState benchState, SolrBenchState solrBenchState) throws SolrServerException, IOException { QueryResponse response = - benchState.longSetQuery(false).process(miniClusterState.client, COLLECTION); + benchState.longSetQuery(false).process(solrBenchState.client, COLLECTION); blackhole.consume(response); return response; } @Benchmark - public Object floatSet( - Blackhole blackhole, - BenchState benchState, - MiniClusterState.MiniClusterBenchState miniClusterState) + public Object floatSet(Blackhole blackhole, BenchState benchState, SolrBenchState solrBenchState) throws SolrServerException, IOException { QueryResponse response = - benchState.floatSetQuery(false).process(miniClusterState.client, COLLECTION); + benchState.floatSetQuery(false).process(solrBenchState.client, COLLECTION); blackhole.consume(response); return response; } @Benchmark - public Object doubleSet( - Blackhole blackhole, - BenchState benchState, - MiniClusterState.MiniClusterBenchState miniClusterState) + public Object doubleSet(Blackhole blackhole, BenchState benchState, SolrBenchState solrBenchState) throws SolrServerException, IOException { QueryResponse response = - benchState.doubleSetQuery(false).process(miniClusterState.client, COLLECTION); + benchState.doubleSetQuery(false).process(solrBenchState.client, COLLECTION); blackhole.consume(response); return response; } @Benchmark - public Object intDvSet( - Blackhole blackhole, - BenchState benchState, - MiniClusterState.MiniClusterBenchState miniClusterState) + public Object intDvSet(Blackhole blackhole, BenchState benchState, SolrBenchState solrBenchState) throws SolrServerException, IOException { QueryResponse response = - benchState.intSetQuery(true).process(miniClusterState.client, COLLECTION); + benchState.intSetQuery(true).process(solrBenchState.client, COLLECTION); blackhole.consume(response); return response; } @Benchmark - public Object longDvSet( - Blackhole blackhole, - BenchState benchState, - MiniClusterState.MiniClusterBenchState miniClusterState) + public Object longDvSet(Blackhole blackhole, BenchState benchState, SolrBenchState solrBenchState) throws SolrServerException, IOException { QueryResponse response = - benchState.longSetQuery(true).process(miniClusterState.client, COLLECTION); + benchState.longSetQuery(true).process(solrBenchState.client, COLLECTION); blackhole.consume(response); return response; } @Benchmark public Object floatDvSet( - Blackhole blackhole, - BenchState benchState, - MiniClusterState.MiniClusterBenchState miniClusterState) + Blackhole blackhole, BenchState benchState, SolrBenchState solrBenchState) throws SolrServerException, IOException { QueryResponse response = - benchState.floatSetQuery(true).process(miniClusterState.client, COLLECTION); + benchState.floatSetQuery(true).process(solrBenchState.client, COLLECTION); blackhole.consume(response); return response; } @Benchmark public Object doubleDvSet( - Blackhole blackhole, - BenchState benchState, - MiniClusterState.MiniClusterBenchState miniClusterState) + Blackhole blackhole, BenchState benchState, SolrBenchState solrBenchState) throws SolrServerException, IOException { QueryResponse response = - benchState.doubleSetQuery(true).process(miniClusterState.client, COLLECTION); + benchState.doubleSetQuery(true).process(solrBenchState.client, COLLECTION); blackhole.consume(response); return response; } diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/QueryResponseWriters.java b/solr/benchmark/src/java/org/apache/solr/bench/search/QueryResponseWriters.java index 4b6118fed27c..6929d2810a57 100644 --- a/solr/benchmark/src/java/org/apache/solr/bench/search/QueryResponseWriters.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/search/QueryResponseWriters.java @@ -23,8 +23,7 @@ import java.io.IOException; import org.apache.solr.bench.Docs; -import org.apache.solr.bench.MiniClusterState; -import org.apache.solr.bench.MiniClusterState.MiniClusterBenchState; +import org.apache.solr.bench.SolrBenchState; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.response.InputStreamResponseParser; @@ -64,10 +63,10 @@ public static class BenchState { private QueryRequest q; @Setup(Level.Trial) - public void setup(MiniClusterBenchState miniClusterState) throws Exception { + public void setup(SolrBenchState solrBenchState) throws Exception { - miniClusterState.startMiniCluster(1); - miniClusterState.createCollection(collection, 1, 1); + solrBenchState.startSolr(1); + solrBenchState.createCollection(collection, 1, 1); // only stored fields are needed to cover the response writers perf Docs docGen = @@ -76,8 +75,8 @@ public void setup(MiniClusterBenchState miniClusterState) throws Exception { .field("text2_ts", strings().basicLatinAlphabet().multi(25).ofLengthBetween(30, 64)) .field("bools_b", booleans().all()) .field("int1_is", integers().all()); - miniClusterState.index(collection, docGen, docs); - miniClusterState.forceMerge(collection, 5); + solrBenchState.index(collection, docGen, docs); + solrBenchState.forceMerge(collection, 5); ModifiableSolrParams params = new ModifiableSolrParams(); params.set(CommonParams.Q, "*:*"); @@ -85,14 +84,13 @@ public void setup(MiniClusterBenchState miniClusterState) throws Exception { params.set(CommonParams.ROWS, docs); q = new QueryRequest(params); q.setResponseParser(new InputStreamResponseParser(wt)); - String base = miniClusterState.nodes.get(0); + String base = solrBenchState.nodes.get(0); } } @Benchmark - public Object query( - BenchState benchState, MiniClusterState.MiniClusterBenchState miniClusterState) + public Object query(BenchState benchState, SolrBenchState solrBenchState) throws SolrServerException, IOException { - return miniClusterState.client.request(benchState.q, collection); + return solrBenchState.client.request(benchState.q, collection); } } diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/SimpleSearch.java b/solr/benchmark/src/java/org/apache/solr/bench/search/SimpleSearch.java index 2e888b8d8788..9159b3221973 100644 --- a/solr/benchmark/src/java/org/apache/solr/bench/search/SimpleSearch.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/search/SimpleSearch.java @@ -19,7 +19,7 @@ import java.io.IOException; import java.util.concurrent.atomic.AtomicLong; import org.apache.solr.bench.BaseBenchState; -import org.apache.solr.bench.MiniClusterState; +import org.apache.solr.bench.SolrBenchState; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.QueryRequest; @@ -60,19 +60,18 @@ public static class BenchState { QueryRequest q = new QueryRequest(new SolrQuery("q", "id:0")); // no match is OK @Setup(Level.Trial) - public void setupTrial(MiniClusterState.MiniClusterBenchState miniClusterState) - throws Exception { - miniClusterState.setUseHttp1(useHttp1); - miniClusterState.startMiniCluster(1); - miniClusterState.createCollection(COLLECTION, 1, 1); + public void setupTrial(SolrBenchState solrBenchState) throws Exception { + solrBenchState.setUseHttp1(useHttp1); + solrBenchState.startSolr(1); + solrBenchState.createCollection(COLLECTION, 1, 1); } @Setup(Level.Iteration) - public void setupIteration(MiniClusterState.MiniClusterBenchState miniClusterState) + public void setupIteration(SolrBenchState solrBenchState) throws SolrServerException, IOException { // Reload the collection/core to drop existing caches CollectionAdminRequest.Reload reload = CollectionAdminRequest.reloadCollection(COLLECTION); - miniClusterState.client.request(reload); + solrBenchState.client.request(reload); total = new AtomicLong(); err = new AtomicLong(); @@ -140,16 +139,15 @@ public void teardownIt() { * } */ @Benchmark - public Object query( - BenchState benchState, MiniClusterState.MiniClusterBenchState miniClusterState, Blackhole bh) + public Object query(BenchState benchState, SolrBenchState solrBenchState, Blackhole bh) throws SolrServerException, IOException { if (benchState.strict) { - return miniClusterState.client.request(benchState.q, COLLECTION); + return solrBenchState.client.request(benchState.q, COLLECTION); } // non strict run ignores exceptions try { - return miniClusterState.client.request(benchState.q, COLLECTION); + return solrBenchState.client.request(benchState.q, COLLECTION); } catch (SolrServerException e) { bh.consume(e); benchState.err.getAndIncrement(); diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/StreamingSearch.java b/solr/benchmark/src/java/org/apache/solr/bench/search/StreamingSearch.java index a9860763dbe7..c354377eaf2f 100644 --- a/solr/benchmark/src/java/org/apache/solr/bench/search/StreamingSearch.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/search/StreamingSearch.java @@ -24,8 +24,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.solr.bench.Docs; -import org.apache.solr.bench.MiniClusterState; -import org.apache.solr.bench.MiniClusterState.MiniClusterBenchState; +import org.apache.solr.bench.SolrBenchState; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; @@ -71,20 +70,20 @@ public static class BenchState { private HttpJettySolrClient httpJettySolrClient; @Setup(Level.Trial) - public void setup(MiniClusterBenchState miniClusterState) throws Exception { + public void setup(SolrBenchState solrBenchState) throws Exception { - miniClusterState.startMiniCluster(3); - miniClusterState.createCollection(collection, 3, 1); + solrBenchState.startSolr(3); + solrBenchState.createCollection(collection, 3, 1); Docs docGen = docs() .field("id", integers().incrementing()) .field("text2_ts", strings().basicLatinAlphabet().multi(312).ofLengthBetween(30, 64)) .field("text3_ts", strings().basicLatinAlphabet().multi(312).ofLengthBetween(30, 64)) .field("int1_i_dv", integers().all()); - miniClusterState.index(collection, docGen, docs); - miniClusterState.waitForMerges(collection); + solrBenchState.index(collection, docGen, docs); + solrBenchState.waitForMerges(collection); - zkHost = miniClusterState.zkHost; + zkHost = solrBenchState.zkHost; params = new ModifiableSolrParams(); params.set(CommonParams.Q, "*:*"); @@ -94,7 +93,7 @@ public void setup(MiniClusterBenchState miniClusterState) throws Exception { } @Setup(Level.Iteration) - public void setupIteration(MiniClusterState.MiniClusterBenchState miniClusterState) + public void setupIteration(SolrBenchState solrBenchState) throws SolrServerException, IOException { SolrClientCache solrClientCache; // TODO tune params? @@ -115,8 +114,7 @@ public void teardownIt() { } @Benchmark - public Object stream( - BenchState benchState, MiniClusterState.MiniClusterBenchState miniClusterState) + public Object stream(BenchState benchState, SolrBenchState solrBenchState) throws SolrServerException, IOException { CloudSolrStream stream = new CloudSolrStream(benchState.zkHost, collection, benchState.params); stream.setStreamContext(benchState.streamContext); diff --git a/solr/benchmark/src/test/org/apache/solr/bench/MiniClusterBenchStateTest.java b/solr/benchmark/src/test/org/apache/solr/bench/SolrBenchStateTest.java similarity index 84% rename from solr/benchmark/src/test/org/apache/solr/bench/MiniClusterBenchStateTest.java rename to solr/benchmark/src/test/org/apache/solr/bench/SolrBenchStateTest.java index 82708f6a34e7..6aabc9657207 100644 --- a/solr/benchmark/src/test/org/apache/solr/bench/MiniClusterBenchStateTest.java +++ b/solr/benchmark/src/test/org/apache/solr/bench/SolrBenchStateTest.java @@ -42,18 +42,18 @@ import org.openjdk.jmh.runner.options.TimeValue; @ThreadLeakLingering(linger = 10) -public class MiniClusterBenchStateTest extends SolrTestCaseJ4 { - private MiniClusterState.MiniClusterBenchState miniBenchState; +public class SolrBenchStateTest extends SolrTestCaseJ4 { + private SolrBenchState solrBenchState; private BaseBenchState baseBenchState; private BenchmarkParams benchParams; @Test - public void testMiniClusterState() throws Exception { + public void test() throws Exception { System.setProperty("workBaseDir", createTempDir("work").toString()); System.setProperty("random.counts", "true"); - miniBenchState = new MiniClusterState.MiniClusterBenchState(); + solrBenchState = new SolrBenchState(); benchParams = new BenchmarkParams( "benchmark", @@ -79,14 +79,14 @@ public void testMiniClusterState() throws Exception { TimeValue.seconds(10)); baseBenchState = new BaseBenchState(); baseBenchState.doSetup(benchParams); - miniBenchState.doSetup(benchParams, baseBenchState); + solrBenchState.doSetup(benchParams, baseBenchState); int nodeCount = 3; - miniBenchState.startMiniCluster(nodeCount); + solrBenchState.startSolr(nodeCount); String collection = "collection1"; int numShards = 1; int numReplicas = 1; - miniBenchState.createCollection(collection, numShards, numReplicas); + solrBenchState.createCollection(collection, numShards, numReplicas); Docs docs = docs() @@ -110,13 +110,13 @@ public void testMiniClusterState() throws Exception { int numDocs = 50; docs.preGenerate(numDocs); - miniBenchState.index(collection, docs, numDocs); + solrBenchState.index(collection, docs, numDocs); - miniBenchState.forceMerge(collection, 15); + solrBenchState.forceMerge(collection, 15); - ModifiableSolrParams params = MiniClusterState.params("q", "*:*"); + ModifiableSolrParams params = SolrBenchState.params("q", "*:*"); QueryRequest queryRequest = new QueryRequest(params); - QueryResponse result = queryRequest.process(miniBenchState.client, collection); + QueryResponse result = queryRequest.process(solrBenchState.client, collection); BaseBenchState.log("match all query result=" + result); @@ -127,7 +127,7 @@ public void testMiniClusterState() throws Exception { public void after() throws Exception { BaseBenchState.doTearDown(benchParams); - miniBenchState.tearDown(benchParams); - miniBenchState.shutdownMiniCluster(benchParams, baseBenchState); + solrBenchState.tearDown(benchParams); + solrBenchState.shutdownSolr(benchParams, baseBenchState); } } diff --git a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java index 96401345503b..fc77cee2e20d 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java @@ -77,6 +77,14 @@ public void close() { syncStrategy.close(); } + /** + * Internally check whether we should abort the election process. This returns true if either this + * context was explicitly closed, or Solr server is being shut down. + */ + private boolean shouldAbort() { + return isClosed || cc.isShutDown(); + } + @Override public void cancelElection() throws InterruptedException, KeeperException { String coreName = leaderProps.getStr(ZkStateReader.CORE_NAME_PROP); @@ -154,7 +162,7 @@ void runLeaderProcess(boolean weAreReplacement) throws KeeperException, Interrup waitForReplicasToComeUp(leaderVoteWait); } - if (isClosed) { + if (shouldAbort()) { // Solr is shutting down or the ZooKeeper session expired while waiting for replicas. If the // later, we cannot be sure we are still the leader, so we should bail out. The OnReconnect // handler will re-register the cores and handle a new leadership election. @@ -185,7 +193,7 @@ void runLeaderProcess(boolean weAreReplacement) throws KeeperException, Interrup } } - if (isClosed) { + if (shouldAbort()) { return; } @@ -267,7 +275,7 @@ void runLeaderProcess(boolean weAreReplacement) throws KeeperException, Interrup } } - if (!isClosed) { + if (!shouldAbort()) { try { if (replicaType.replicateFromLeader) { // stop replicate from old leader @@ -361,7 +369,7 @@ private boolean waitForEligibleBecomeLeaderAfterTimeout( ZkShardTerms zkShardTerms, String coreNodeName, int timeout) throws InterruptedException { long timeoutAt = System.nanoTime() + TimeUnit.NANOSECONDS.convert(timeout, TimeUnit.MILLISECONDS); - while (!isClosed && !cc.isShutDown()) { + while (!shouldAbort()) { if (System.nanoTime() > timeoutAt) { log.warn( "After waiting for {}ms, no other potential leader was found, {} try to become leader anyway (core_term:{}, highest_term:{})", @@ -446,7 +454,7 @@ private boolean waitForReplicasToComeUp(int timeoutms) throws InterruptedExcepti DocCollection docCollection = zkController.getClusterState().getCollectionOrNull(collection); Slice slices = (docCollection == null) ? null : docCollection.getSlice(shardId); int cnt = 0; - while (!isClosed && !cc.isShutDown()) { + while (!shouldAbort()) { // wait for everyone to be up if (slices != null) { int found = 0; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/HealthCheckHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/HealthCheckHandler.java index 1ecf959e49ed..1dab5d1d9778 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/HealthCheckHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/HealthCheckHandler.java @@ -17,39 +17,21 @@ package org.apache.solr.handler.admin; -import static org.apache.solr.common.params.CommonParams.FAILURE; -import static org.apache.solr.common.params.CommonParams.OK; -import static org.apache.solr.common.params.CommonParams.STATUS; -import static org.apache.solr.handler.admin.api.ReplicationAPIBase.GENERATION; - -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.List; -import java.util.Locale; -import java.util.stream.Collectors; -import org.apache.lucene.index.IndexCommit; -import org.apache.solr.api.AnnotatedApi; import org.apache.solr.api.Api; +import org.apache.solr.api.JerseyResource; +import org.apache.solr.client.api.model.NodeHealthResponse; import org.apache.solr.client.solrj.request.HealthCheckRequest; -import org.apache.solr.cloud.CloudDescriptor; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.Replica.State; -import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.CoreContainer; -import org.apache.solr.core.SolrCore; -import org.apache.solr.handler.IndexFetcher; -import org.apache.solr.handler.ReplicationHandler; import org.apache.solr.handler.RequestHandlerBase; -import org.apache.solr.handler.admin.api.NodeHealthAPI; +import org.apache.solr.handler.admin.api.NodeHealth; +import org.apache.solr.handler.api.V2ApiUtils; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.security.AuthorizationContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Health Check Handler for reporting the health of a specific node. @@ -77,12 +59,13 @@ * specify the acceptable generation lag follower should be with respect to its leader using the * maxGenerationLag=<max_generation_lag> request parameter. If * maxGenerationLag is not provided then health check would simply return OK. + * + *

All health-check logic lives in the v2 {@link NodeHealth}; this handler is a thin v1 bridge + * that extracts request parameters and delegates. */ public class HealthCheckHandler extends RequestHandlerBase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final String PARAM_REQUIRE_HEALTHY_CORES = "requireHealthyCores"; - private static final List UNHEALTHY_STATES = Arrays.asList(State.DOWN, State.RECOVERING); CoreContainer coreContainer; @@ -100,224 +83,18 @@ public CoreContainer getCoreContainer() { @Override public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { rsp.setHttpCaching(false); - - // Core container should not be null and active (redundant check) - if (coreContainer == null || coreContainer.isShutDown()) { - rsp.setException( - new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "CoreContainer is either not initialized or shutting down")); - return; - } - if (!coreContainer.isZooKeeperAware()) { - if (log.isDebugEnabled()) { - log.debug("Invoked HealthCheckHandler in legacy mode."); - } - healthCheckLegacyMode(req, rsp); - } else { - if (log.isDebugEnabled()) { - log.debug( - "Invoked HealthCheckHandler in cloud mode on [{}]", - this.coreContainer.getZkController().getNodeName()); - } - healthCheckCloudMode(req, rsp); - } - } - - private void healthCheckCloudMode(SolrQueryRequest req, SolrQueryResponse rsp) { - ZkStateReader zkStateReader = coreContainer.getZkController().getZkStateReader(); - ClusterState clusterState = zkStateReader.getClusterState(); - // Check for isConnected and isClosed - if (zkStateReader.getZkClient().isClosed() || !zkStateReader.getZkClient().isConnected()) { - rsp.add(STATUS, FAILURE); - rsp.setException( - new SolrException( - SolrException.ErrorCode.SERVICE_UNAVAILABLE, - "Host Unavailable: Not connected to zk")); - return; - } - - // Fail if not in live_nodes - if (!clusterState.getLiveNodes().contains(coreContainer.getZkController().getNodeName())) { - rsp.add(STATUS, FAILURE); - rsp.setException( - new SolrException( - SolrException.ErrorCode.SERVICE_UNAVAILABLE, - "Host Unavailable: Not in live nodes as per zk")); - return; - } - - // Optionally require that all cores on this node are active if param 'requireHealthyCores=true' - if (req.getParams().getBool(PARAM_REQUIRE_HEALTHY_CORES, false)) { - if (!coreContainer.isStatusLoadComplete()) { - rsp.add(STATUS, FAILURE); - rsp.setException( - new SolrException( - SolrException.ErrorCode.SERVICE_UNAVAILABLE, - "Host Unavailable: Core Loading not complete")); - return; - } - Collection coreDescriptors = - coreContainer.getCoreDescriptors().stream() - .map(cd -> cd.getCloudDescriptor()) - .collect(Collectors.toList()); - long unhealthyCores = findUnhealthyCores(coreDescriptors, clusterState); - if (unhealthyCores > 0) { - rsp.add(STATUS, FAILURE); - rsp.add("num_cores_unhealthy", unhealthyCores); - rsp.setException( - new SolrException( - SolrException.ErrorCode.SERVICE_UNAVAILABLE, - unhealthyCores - + " out of " - + coreContainer.getNumAllCores() - + " replicas are currently initializing or recovering")); - return; - } - rsp.add("message", "All cores are healthy"); - } - - // All lights green, report healthy - rsp.add(STATUS, OK); - } - - private void healthCheckLegacyMode(SolrQueryRequest req, SolrQueryResponse rsp) { - Integer maxGenerationLag = req.getParams().getInt(HealthCheckRequest.PARAM_MAX_GENERATION_LAG); - List laggingCoresInfo = new ArrayList<>(); - boolean allCoresAreInSync = true; - - // check only if max generation lag is specified - if (maxGenerationLag != null) { - // if is not negative - if (maxGenerationLag < 0) { - log.error("Invalid value for maxGenerationLag:[{}]", maxGenerationLag); - rsp.add( - "message", - String.format(Locale.ROOT, "Invalid value of maxGenerationLag:%s", maxGenerationLag)); - rsp.add(STATUS, FAILURE); - } else { - for (SolrCore core : coreContainer.getCores()) { - ReplicationHandler replicationHandler = - (ReplicationHandler) core.getRequestHandler(ReplicationHandler.PATH); - if (replicationHandler.isFollower()) { - boolean isCoreInSync = - isWithinGenerationLag(core, replicationHandler, maxGenerationLag, laggingCoresInfo); - - allCoresAreInSync &= isCoreInSync; - } - } - } - if (allCoresAreInSync) { - rsp.add( - "message", - String.format( - Locale.ROOT, - "All the followers are in sync with leader (within maxGenerationLag: %d) " - + "or the cores are acting as leader", - maxGenerationLag)); - rsp.add(STATUS, OK); - } else { - rsp.add( - "message", - String.format( - Locale.ROOT, - "Cores violating maxGenerationLag:%d.%n%s", - maxGenerationLag, - String.join(",\n", laggingCoresInfo))); - rsp.add(STATUS, FAILURE); - } - } else { // if maxGeneration lag is not specified (is null) we aren't checking for lag - rsp.add( - "message", - "maxGenerationLag isn't specified. Followers aren't " - + "checking for the generation lag from the leaders"); - rsp.add(STATUS, OK); - } - } - - private boolean isWithinGenerationLag( - final SolrCore core, - ReplicationHandler replicationHandler, - int maxGenerationLag, - List laggingCoresInfo) { - IndexFetcher indexFetcher = null; + final Boolean requireHealthyCores = req.getParams().getBool(PARAM_REQUIRE_HEALTHY_CORES); + final Integer maxGenerationLag = + req.getParams().getInt(HealthCheckRequest.PARAM_MAX_GENERATION_LAG); try { - // may not be the best way to get leader's replicableCommit - NamedList follower = (NamedList) replicationHandler.getInitArgs().get("follower"); - - indexFetcher = new IndexFetcher(follower, replicationHandler, core); - - NamedList replicableCommitOnLeader = indexFetcher.getLatestVersion(); - long leaderGeneration = (Long) replicableCommitOnLeader.get(GENERATION); - - // Get our own commit and generation from the commit - IndexCommit commit = core.getDeletionPolicy().getLatestCommit(); - if (commit != null) { - long followerGeneration = commit.getGeneration(); - long generationDiff = leaderGeneration - followerGeneration; - - // generationDiff shouldn't be negative except for some edge cases, log it. Some scenarios - // are - // 1) commit generation rolls over Long.MAX_VALUE (really unlikely) - // 2) Leader's index is wiped clean and the follower is still showing commit generation - // from the old index - if (generationDiff < 0) { - log.warn("core:[{}], generation lag:[{}] is negative."); - } else if (generationDiff < maxGenerationLag) { - log.info( - "core:[{}] generation lag is above acceptable threshold:[{}], " - + "generation lag:[{}], leader generation:[{}], follower generation:[{}]", - core, - maxGenerationLag, - generationDiff, - leaderGeneration, - followerGeneration); - - laggingCoresInfo.add( - String.format( - Locale.ROOT, - "Core %s is lagging by %d generations", - core.getName(), - generationDiff)); - return true; - } - } - } catch (Exception e) { - log.error("Failed to check if the follower is in sync with the leader", e); - } finally { - if (indexFetcher != null) { - indexFetcher.destroy(); - } + V2ApiUtils.squashIntoSolrResponseWithoutHeader( + rsp, new NodeHealth(coreContainer).healthcheck(requireHealthyCores, maxGenerationLag)); + } catch (SolrException e) { + final NodeHealthResponse failureResponse = new NodeHealthResponse(); + failureResponse.status = NodeHealthResponse.NodeStatus.FAILURE; + V2ApiUtils.squashIntoSolrResponseWithoutHeader(rsp, failureResponse); + rsp.setException(e); } - return false; - } - - /** - * Find replicas DOWN or RECOVERING, or replicas in clusterstate that do not exist on local node. - * We first find local cores which are either not registered or unhealthy, and check each of these - * against the clusterstate, and return a count of unhealthy replicas - * - * @param cores list of core cloud descriptors to iterate - * @param clusterState clusterstate from ZK - * @return number of unhealthy cores, either in DOWN or RECOVERING state - */ - static long findUnhealthyCores(Collection cores, ClusterState clusterState) { - return cores.stream() - .filter( - c -> - !c.hasRegistered() - || UNHEALTHY_STATES.contains(c.getLastPublished())) // Find candidates locally - .filter( - c -> - clusterState.hasCollection( - c.getCollectionName())) // Only care about cores for actual collections - .filter( - c -> - clusterState - .getCollection(c.getCollectionName()) - .getActiveSlicesMap() - .containsKey(c.getShardId())) - .count(); } @Override @@ -337,7 +114,12 @@ public Boolean registerV2() { @Override public Collection getApis() { - return AnnotatedApi.getApis(new NodeHealthAPI(this)); + return List.of(); + } + + @Override + public Collection> getJerseyResources() { + return List.of(NodeHealth.class); } @Override diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/NodeHealth.java b/solr/core/src/java/org/apache/solr/handler/admin/api/NodeHealth.java new file mode 100644 index 000000000000..925652d2db06 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/NodeHealth.java @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.admin.api; + +import static org.apache.solr.client.api.model.NodeHealthResponse.NodeStatus.FAILURE; +import static org.apache.solr.client.api.model.NodeHealthResponse.NodeStatus.OK; +import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR; +import static org.apache.solr.common.SolrException.ErrorCode.SERVICE_UNAVAILABLE; +import static org.apache.solr.handler.admin.api.ReplicationAPIBase.GENERATION; +import static org.apache.solr.security.PermissionNameProvider.Name.HEALTH_PERM; + +import jakarta.inject.Inject; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; +import org.apache.lucene.index.IndexCommit; +import org.apache.solr.api.JerseyResource; +import org.apache.solr.client.api.endpoint.NodeHealthApi; +import org.apache.solr.client.api.model.NodeHealthResponse; +import org.apache.solr.cloud.CloudDescriptor; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.Replica.State; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.CoreDescriptor; +import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.IndexFetcher; +import org.apache.solr.handler.ReplicationHandler; +import org.apache.solr.jersey.PermissionName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * V2 API for checking the health of the receiving node. + * + *

This API (GET /v2/node/health) is analogous to the v1 /admin/info/health. + * + *

The v1 {@link org.apache.solr.handler.admin.HealthCheckHandler} delegates to this class. + */ +public class NodeHealth extends JerseyResource implements NodeHealthApi { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final List UNHEALTHY_STATES = Arrays.asList(State.DOWN, State.RECOVERING); + + private final CoreContainer coreContainer; + + @Inject + public NodeHealth(CoreContainer coreContainer) { + this.coreContainer = coreContainer; + } + + @Override + @PermissionName(HEALTH_PERM) + public NodeHealthResponse healthcheck(Boolean requireHealthyCores, Integer maxGenerationLag) { + if (coreContainer == null || coreContainer.isShutDown()) { + throw new SolrException( + SERVER_ERROR, "CoreContainer is either not initialized or shutting down"); + } + + final NodeHealthResponse response = instantiateJerseyResponse(NodeHealthResponse.class); + + if (!coreContainer.isZooKeeperAware()) { + if (log.isDebugEnabled()) { + log.debug("Invoked HealthCheckHandler in legacy mode."); + } + healthCheckStandaloneMode(response, maxGenerationLag); + } else { + if (log.isDebugEnabled()) { + log.debug( + "Invoked HealthCheckHandler in cloud mode on [{}]", + coreContainer.getZkController().getNodeName()); + } + healthCheckCloudMode(response, requireHealthyCores); + } + + return response; + } + + private void healthCheckCloudMode(NodeHealthResponse response, Boolean requireHealthyCores) { + ClusterState clusterState = getClusterState(); + + if (Boolean.TRUE.equals(requireHealthyCores)) { + if (!coreContainer.isStatusLoadComplete()) { + throw new SolrException(SERVICE_UNAVAILABLE, "Host Unavailable: Core Loading not complete"); + } + Collection coreDescriptors = + coreContainer.getCoreDescriptors().stream() + .map(CoreDescriptor::getCloudDescriptor) + .collect(Collectors.toList()); + int unhealthyCores = findUnhealthyCores(coreDescriptors, clusterState); + if (unhealthyCores > 0) { + response.numCoresUnhealthy = unhealthyCores; + throw new SolrException( + SERVICE_UNAVAILABLE, + unhealthyCores + + " out of " + + coreContainer.getNumAllCores() + + " replicas are currently initializing or recovering"); + } + response.message = "All cores are healthy"; + } + + response.status = OK; + } + + private ClusterState getClusterState() { + ZkStateReader zkStateReader = coreContainer.getZkController().getZkStateReader(); + ClusterState clusterState = zkStateReader.getClusterState(); + + if (zkStateReader.getZkClient().isClosed() || !zkStateReader.getZkClient().isConnected()) { + throw new SolrException(SERVICE_UNAVAILABLE, "Host Unavailable: Not connected to zk"); + } + + if (!clusterState.getLiveNodes().contains(coreContainer.getZkController().getNodeName())) { + throw new SolrException(SERVICE_UNAVAILABLE, "Host Unavailable: Not in live nodes as per zk"); + } + return clusterState; + } + + private void healthCheckStandaloneMode(NodeHealthResponse response, Integer maxGenerationLag) { + List laggingCoresInfo = new ArrayList<>(); + boolean allCoresAreInSync = true; + + if (maxGenerationLag != null) { + if (maxGenerationLag < 0) { + log.error("Invalid value for maxGenerationLag:[{}]", maxGenerationLag); + response.message = + String.format(Locale.ROOT, "Invalid value of maxGenerationLag:%s", maxGenerationLag); + response.status = FAILURE; + return; + } + + for (SolrCore core : coreContainer.getCores()) { + ReplicationHandler replicationHandler = + (ReplicationHandler) core.getRequestHandler(ReplicationHandler.PATH); + if (replicationHandler.isFollower()) { + boolean isCoreInSync = + isWithinGenerationLag(core, replicationHandler, maxGenerationLag, laggingCoresInfo); + allCoresAreInSync &= isCoreInSync; + } + } + + if (allCoresAreInSync) { + response.message = + String.format( + Locale.ROOT, + "All the followers are in sync with leader (within maxGenerationLag: %d) " + + "or the cores are acting as leader", + maxGenerationLag); + response.status = OK; + } else { + response.message = + String.format( + Locale.ROOT, + "Cores violating maxGenerationLag:%d.%n%s", + maxGenerationLag, + String.join(",\n", laggingCoresInfo)); + response.status = FAILURE; + } + } else { + response.message = + "maxGenerationLag isn't specified. Followers aren't " + + "checking for the generation lag from the leaders"; + response.status = OK; + } + } + + private boolean isWithinGenerationLag( + final SolrCore core, + ReplicationHandler replicationHandler, + int maxGenerationLag, + List laggingCoresInfo) { + IndexFetcher indexFetcher = null; + try { + // may not be the best way to get leader's replicableCommit; NamedList is unavoidable here + // as it is the init-args format used by ReplicationHandler + NamedList follower = (NamedList) replicationHandler.getInitArgs().get("follower"); + indexFetcher = new IndexFetcher(follower, replicationHandler, core); + // getLatestVersion() returns a NamedList from the IndexFetcher network API + NamedList replicableCommitOnLeader = indexFetcher.getLatestVersion(); + long leaderGeneration = (Long) replicableCommitOnLeader.get(GENERATION); + + // Get our own commit and generation from the commit + IndexCommit commit = core.getDeletionPolicy().getLatestCommit(); + if (commit != null) { + long followerGeneration = commit.getGeneration(); + long generationDiff = leaderGeneration - followerGeneration; + + // generationDiff shouldn't be negative except for some edge cases, log it. Some scenarios + // are: + // 1) commit generation rolls over Long.MAX_VALUE (really unlikely) + // 2) Leader's index is wiped clean and the follower is still showing commit generation + // from the old index + if (generationDiff < 0) { + log.warn("core:[{}], generation lag:[{}] is negative.", core, generationDiff); + } else if (generationDiff > maxGenerationLag) { + log.info( + "core:[{}] generation lag is above acceptable threshold:[{}], " + + "generation lag:[{}], leader generation:[{}], follower generation:[{}]", + core, + maxGenerationLag, + generationDiff, + leaderGeneration, + followerGeneration); + laggingCoresInfo.add( + String.format( + Locale.ROOT, + "Core %s is lagging by %d generations", + core.getName(), + generationDiff)); + return false; + } + } + } catch (Exception e) { + log.error("Failed to check if the follower is in sync with the leader", e); + return false; + } finally { + if (indexFetcher != null) { + indexFetcher.destroy(); + } + } + return true; + } + + /** + * Find replicas DOWN or RECOVERING, or replicas in clusterstate that do not exist on local node. + * We first find local cores which are either not registered or unhealthy, and check each of these + * against the clusterstate, and return a count of unhealthy replicas. + * + * @param cores list of core cloud descriptors to iterate + * @param clusterState clusterstate from ZK + * @return number of unhealthy cores, either in DOWN or RECOVERING state + */ + public static int findUnhealthyCores( + Collection cores, ClusterState clusterState) { + return Math.toIntExact( + cores.stream() + .filter( + c -> + !c.hasRegistered() + || UNHEALTHY_STATES.contains( + c.getLastPublished())) // Find candidates locally + .filter( + c -> + clusterState.hasCollection( + c.getCollectionName())) // Only care about cores for actual collections + .filter( + c -> + clusterState + .getCollection(c.getCollectionName()) + .getActiveSlicesMap() + .containsKey(c.getShardId())) + .count()); + } +} diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/NodeHealthAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/NodeHealthAPI.java deleted file mode 100644 index df5f64900f03..000000000000 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/NodeHealthAPI.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.handler.admin.api; - -import static org.apache.solr.client.solrj.SolrRequest.METHOD.GET; -import static org.apache.solr.security.PermissionNameProvider.Name.HEALTH_PERM; - -import org.apache.solr.api.EndPoint; -import org.apache.solr.handler.admin.HealthCheckHandler; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; - -/** - * V2 API for checking the health of the receiving node. - * - *

This API (GET /v2/node/health) is analogous to the v1 /admin/info/health. - */ -public class NodeHealthAPI { - private final HealthCheckHandler handler; - - public NodeHealthAPI(HealthCheckHandler handler) { - this.handler = handler; - } - - // TODO Update permission here once SOLR-11623 lands. - @EndPoint( - path = {"/node/health"}, - method = GET, - permission = HEALTH_PERM) - public void getSystemInformation(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { - handler.handleRequestBody(req, rsp); - } -} diff --git a/solr/core/src/java/org/apache/solr/metrics/OtelRuntimeJvmMetrics.java b/solr/core/src/java/org/apache/solr/metrics/OtelRuntimeJvmMetrics.java index 19803372dae9..a9c14bab2267 100644 --- a/solr/core/src/java/org/apache/solr/metrics/OtelRuntimeJvmMetrics.java +++ b/solr/core/src/java/org/apache/solr/metrics/OtelRuntimeJvmMetrics.java @@ -16,13 +16,20 @@ */ package org.apache.solr.metrics; +import static org.apache.solr.metrics.SolrMetricProducer.STATE_KEY_ATTR; + import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.metrics.MeterProvider; +import io.opentelemetry.api.metrics.ObservableLongGauge; import io.opentelemetry.api.trace.TracerProvider; import io.opentelemetry.context.propagation.ContextPropagators; import io.opentelemetry.instrumentation.runtimemetrics.java17.RuntimeMetrics; import java.lang.invoke.MethodHandles; +import java.lang.management.ManagementFactory; +import org.apache.lucene.util.SuppressForbidden; import org.apache.solr.common.util.EnvUtils; +import org.apache.solr.metrics.otel.OtelUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,6 +38,7 @@ public class OtelRuntimeJvmMetrics { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private RuntimeMetrics runtimeMetrics; + private ObservableLongGauge systemMemoryGauge; private boolean isInitialized = false; // Main feature flag to enable/disable all JVM metrics @@ -38,6 +46,10 @@ public static boolean isJvmMetricsEnabled() { return EnvUtils.getPropertyAsBool("solr.metrics.jvm.enabled", true); } + @SuppressForbidden( + reason = + "com.sun.management.OperatingSystemMXBean is used intentionally for physical memory" + + " gauges; guarded by instanceof check so gracefully absent on non-HotSpot JVMs") public OtelRuntimeJvmMetrics initialize( SolrMetricManager solrMetricManager, String registryName) { if (!isJvmMetricsEnabled()) return this; @@ -65,6 +77,30 @@ public ContextPropagators getPropagators() { // TODO: We should have this configurable to enable/disable specific JVM metrics .enableAllFeatures() .build(); + java.lang.management.OperatingSystemMXBean osMxBean = + ManagementFactory.getOperatingSystemMXBean(); + if (osMxBean instanceof com.sun.management.OperatingSystemMXBean extOsMxBean) { + systemMemoryGauge = + solrMetricManager.observableLongGauge( + registryName, + "jvm.system.memory", + "Physical memory of the host or container in bytes (state=total|free)." + + " On Linux with cgroup limits, total reflects the container memory limit.", + measurement -> { + long total = extOsMxBean.getTotalMemorySize(); + long free = extOsMxBean.getFreeMemorySize(); + if (total >= 0) measurement.record(total, Attributes.of(STATE_KEY_ATTR, "total")); + if (free >= 0) measurement.record(free, Attributes.of(STATE_KEY_ATTR, "free")); + }, + OtelUnit.BYTES); + log.info("Physical memory metrics enabled"); + } else { + if (log.isDebugEnabled()) { + log.debug( + "Physical memory metrics unavailable:" + + " com.sun.management.OperatingSystemMXBean not present on this JVM"); + } + } isInitialized = true; log.info("JVM metrics collection successfully initialized"); return this; @@ -74,6 +110,10 @@ public void close() { if (runtimeMetrics != null && isInitialized) { try { runtimeMetrics.close(); + if (systemMemoryGauge != null) { + systemMemoryGauge.close(); + systemMemoryGauge = null; + } } catch (Exception e) { log.error("Failed to close JVM metrics collection", e); } finally { diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricProducer.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricProducer.java index 9631c5bedfd5..d98545b36df7 100644 --- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricProducer.java +++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricProducer.java @@ -30,6 +30,7 @@ public interface SolrMetricProducer extends AutoCloseable { public static final AttributeKey RESULT_ATTR = AttributeKey.stringKey("result"); public static final AttributeKey NAME_ATTR = AttributeKey.stringKey("name"); public static final AttributeKey PLUGIN_NAME_ATTR = AttributeKey.stringKey("plugin_name"); + public static final AttributeKey STATE_KEY_ATTR = AttributeKey.stringKey("state"); /** * Unique metric tag identifies components with the same life-cycle, which should be registered / diff --git a/solr/core/src/java/org/apache/solr/search/join/CrossCollectionJoinQParser.java b/solr/core/src/java/org/apache/solr/search/join/CrossCollectionJoinQParser.java index 1b78a0cb2e07..2e4675a880d2 100644 --- a/solr/core/src/java/org/apache/solr/search/join/CrossCollectionJoinQParser.java +++ b/solr/core/src/java/org/apache/solr/search/join/CrossCollectionJoinQParser.java @@ -23,6 +23,7 @@ import java.util.Set; import org.apache.lucene.search.Query; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.ShardParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.search.QParser; @@ -102,6 +103,12 @@ public Query parse() throws SyntaxError { } } + // Propagate shards.preference from request-level params if not already set in localParams + String shardsPreference = req.getParams().get(ShardParams.SHARDS_PREFERENCE); + if (shardsPreference != null && otherParams.get(ShardParams.SHARDS_PREFERENCE) == null) { + otherParams.set(ShardParams.SHARDS_PREFERENCE, shardsPreference); + } + return new CrossCollectionJoinQuery( query, zkHost, solrUrl, collection, fromField, toField, routedByJoinKey, ttl, otherParams); } diff --git a/solr/core/src/java/org/apache/solr/search/join/CrossCollectionJoinQuery.java b/solr/core/src/java/org/apache/solr/search/join/CrossCollectionJoinQuery.java index bb03022afdfd..8381ed17dbb6 100644 --- a/solr/core/src/java/org/apache/solr/search/join/CrossCollectionJoinQuery.java +++ b/solr/core/src/java/org/apache/solr/search/join/CrossCollectionJoinQuery.java @@ -48,11 +48,14 @@ import org.apache.solr.client.solrj.io.stream.UniqueStream; import org.apache.solr.client.solrj.io.stream.expr.StreamExpression; import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter; +import org.apache.solr.client.solrj.routing.RequestReplicaListTransformerGenerator; import org.apache.solr.cloud.CloudDescriptor; +import org.apache.solr.cloud.ZkController; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocRouter; import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; @@ -219,11 +222,13 @@ private String createHashRangeFq() { } private TupleStream createCloudSolrStream(SolrClientCache solrClientCache) throws IOException { + ZkController zkController = searcher.getCore().getCoreContainer().getZkController(); + String streamZkHost; if (zkHost != null) { streamZkHost = zkHost; } else { - streamZkHost = searcher.getCore().getCoreContainer().getZkController().getZkServerAddress(); + streamZkHost = zkController.getZkServerAddress(); } ModifiableSolrParams params = new ModifiableSolrParams(otherParams); @@ -239,6 +244,21 @@ private TupleStream createCloudSolrStream(SolrClientCache solrClientCache) throw StreamContext streamContext = new StreamContext(); streamContext.setSolrClientCache(solrClientCache); + streamContext.setRequestParams(new ModifiableSolrParams(otherParams)); + if (zkController != null) { + RequestReplicaListTransformerGenerator rltg = + new RequestReplicaListTransformerGenerator( + zkController + .getZkStateReader() + .getClusterProperties() + .getOrDefault(ZkStateReader.DEFAULT_SHARD_PREFERENCES, "") + .toString(), + zkController.getNodeName(), + zkController.getBaseUrl(), + zkController.getHostName(), + zkController.getSysPropsCacher()); + streamContext.setRequestReplicaListTransformerGenerator(rltg); + } TupleStream cloudSolrStream = new CloudSolrStream(streamZkHost, collection, params); TupleStream uniqueStream = new UniqueStream(cloudSolrStream, new FieldEqualitor(fromField)); diff --git a/solr/core/src/test/org/apache/solr/TestTolerantSearch.java b/solr/core/src/test/org/apache/solr/TestTolerantSearch.java index 5c5b5b634709..1c3e23aa613c 100644 --- a/solr/core/src/test/org/apache/solr/TestTolerantSearch.java +++ b/solr/core/src/test/org/apache/solr/TestTolerantSearch.java @@ -70,8 +70,7 @@ private static Path createSolrHome() throws Exception { @BeforeClass public static void createThings() throws Exception { systemSetPropertyEnableUrlAllowList(false); - Path solrHome = createSolrHome(); - solrTestRule.startSolr(solrHome); + solrTestRule.startSolr(createSolrHome()); collection1 = solrTestRule.getSolrClient("collection1"); diff --git a/solr/core/src/test/org/apache/solr/handler/TestHttpRequestId.java b/solr/core/src/test/org/apache/solr/handler/TestHttpRequestId.java index fc560425209b..ef0f486fa30e 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestHttpRequestId.java +++ b/solr/core/src/test/org/apache/solr/handler/TestHttpRequestId.java @@ -48,7 +48,7 @@ public class TestHttpRequestId extends SolrTestCaseJ4 { @BeforeClass public static void beforeTest() throws Exception { - solrTestRule.startSolr(createTempDir()); + solrTestRule.startSolr(); } @Test diff --git a/solr/core/src/test/org/apache/solr/handler/admin/HealthCheckHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/HealthCheckHandlerTest.java index 43838707d057..79036e5c16ed 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/HealthCheckHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/HealthCheckHandlerTest.java @@ -18,11 +18,17 @@ package org.apache.solr.handler.admin; import static org.apache.solr.common.params.CommonParams.HEALTH_CHECK_HANDLER_PATH; +import static org.hamcrest.Matchers.containsString; import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; import java.util.Arrays; import java.util.Collection; import java.util.Properties; +import java.util.concurrent.TimeUnit; import org.apache.solr.client.solrj.RemoteSolrException; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; @@ -30,10 +36,8 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.GenericSolrRequest; import org.apache.solr.client.solrj.request.HealthCheckRequest; -import org.apache.solr.client.solrj.request.V2Request; import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.apache.solr.client.solrj.response.HealthCheckResponse; -import org.apache.solr.client.solrj.response.V2Response; import org.apache.solr.cloud.CloudDescriptor; import org.apache.solr.cloud.ClusterStateMockUtil; import org.apache.solr.cloud.SolrCloudTestCase; @@ -44,6 +48,7 @@ import org.apache.solr.common.params.CommonParams; import org.apache.solr.core.CoreDescriptor; import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.handler.admin.api.NodeHealth; import org.junit.BeforeClass; import org.junit.Test; @@ -104,12 +109,8 @@ public void testHealthCheckHandler() throws Exception { // negative check of our (new) "broken" node that we deliberately put into an unhealthy state RemoteSolrException e = - expectThrows( - RemoteSolrException.class, - () -> { - runHealthcheckWithClient(solrClient); - }); - assertTrue(e.getMessage(), e.getMessage().contains("Host Unavailable")); + expectThrows(RemoteSolrException.class, () -> runHealthcheckWithClient(solrClient)); + assertThat(e.getMessage(), containsString("Host Unavailable")); assertEquals(SolrException.ErrorCode.SERVICE_UNAVAILABLE.code, e.code()); } finally { newJetty.stop(); @@ -135,37 +136,56 @@ public void testHealthCheckHandlerSolrJ() throws IOException, SolrServerExceptio } } + /** + * Verifies that the v1 health-check response body contains {@code "status":"FAILURE"} when the + * node is absent from ZooKeeper's live-nodes set. + * + *

This is a regression test for the refactoring that delegated health-check logic to {@link + * NodeHealth}: after that change, {@link SolrException} thrown by {@link NodeHealth} would escape + * {@link HealthCheckHandler#handleRequestBody} before the {@code status} field was written to the + * response, leaving callers without a machine-readable failure indicator in the body. + * + *

The node's ZK session is kept alive so that only the live-nodes check fires, not the "not + * connected to ZK" check, isolating the specific code path under test. + */ @Test - public void testHealthCheckV2Api() throws Exception { - V2Response res = new V2Request.Builder("/node/health").build().process(cluster.getSolrClient()); - assertEquals(0, res.getStatus()); - assertEquals(CommonParams.OK, res.getResponse().get(CommonParams.STATUS)); - - // add a new node for the purpose of negative testing + public void testV1FailureResponseIncludesStatusField() throws Exception { JettySolrRunner newJetty = cluster.startJettySolrRunner(); try (SolrClient solrClient = getHttpSolrClient(newJetty.getBaseUrl().toString())) { + // Sanity check: the new node is initially healthy. + assertEquals(CommonParams.OK, runHealthcheckWithClient(solrClient).getNodeStatus()); - // positive check that our (new) "healthy" node works with direct http client - assertEquals( - CommonParams.OK, - new V2Request.Builder("/node/health") - .build() - .process(solrClient) - .getResponse() - .get(CommonParams.STATUS)); - - // now "break" our (new) node - newJetty.getCoreContainer().getZkController().getZkClient().close(); - - // negative check of our (new) "broken" node that we deliberately put into an unhealthy state - RemoteSolrException e = - expectThrows( - RemoteSolrException.class, - () -> { - new V2Request.Builder("/node/health").build().process(solrClient); - }); - assertTrue(e.getMessage(), e.getMessage().contains("Host Unavailable")); - assertEquals(SolrException.ErrorCode.SERVICE_UNAVAILABLE.code, e.code()); + String nodeName = newJetty.getCoreContainer().getZkController().getNodeName(); + + // Remove the node from ZooKeeper's live_nodes without closing the ZK session. + // This ensures the "ZK not connected" check passes and only the "not in live nodes" + // check fires, exercising the specific failure branch we fixed. + newJetty.getCoreContainer().getZkController().removeEphemeralLiveNode(); + + // Wait for the node's own ZkStateReader to reflect the removal before querying. + newJetty + .getCoreContainer() + .getZkController() + .getZkStateReader() + .waitForLiveNodes(10, TimeUnit.SECONDS, missingLiveNode(nodeName)); + + // Use a raw HTTP request so we can inspect the full response body. + // SolrJ's HealthCheckRequest throws RemoteSolrException on non-200 responses and does + // not expose the response body, so we go below SolrJ here. + try (HttpClient httpClient = HttpClient.newHttpClient()) { + HttpResponse response = + httpClient.send( + HttpRequest.newBuilder() + .uri(URI.create(newJetty.getBaseUrl() + HEALTH_CHECK_HANDLER_PATH)) + .build(), + HttpResponse.BodyHandlers.ofString()); + + assertEquals("Expected 503 SERVICE_UNAVAILABLE", 503, response.statusCode()); + assertThat( + "v1 error response body must contain status=FAILURE so body-inspecting clients get a clear signal", + response.body(), + containsString("FAILURE")); + } } finally { newJetty.stop(); } @@ -193,7 +213,7 @@ public void testFindUnhealthyCores() { mockCD("invalid", "invalid", "slice1", false, Replica.State.RECOVERING), // A core for a slice that is not an active slice will not fail the check mockCD("collection1", "invalid_replica1", "invalid", true, Replica.State.DOWN)); - long unhealthy1 = HealthCheckHandler.findUnhealthyCores(node1Cores, clusterState); + long unhealthy1 = NodeHealth.findUnhealthyCores(node1Cores, clusterState); assertEquals(2, unhealthy1); // Node 2 @@ -203,7 +223,7 @@ public void testFindUnhealthyCores() { mockCD("collection1", "slice1_replica4", "slice1", true, Replica.State.DOWN), mockCD( "collection2", "slice1_replica1", "slice1", true, Replica.State.RECOVERY_FAILED)); - long unhealthy2 = HealthCheckHandler.findUnhealthyCores(node2Cores, clusterState); + long unhealthy2 = NodeHealth.findUnhealthyCores(node2Cores, clusterState); assertEquals(1, unhealthy2); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/ShowFileRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/ShowFileRequestHandlerTest.java index a22f78431373..69f3108ba8a1 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/ShowFileRequestHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/ShowFileRequestHandlerTest.java @@ -54,11 +54,8 @@ public class ShowFileRequestHandlerTest extends SolrTestCaseJ4 { public static void beforeTest() throws Exception { EnvUtils.setProperty( ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME.toAbsolutePath().toString()); - solrTestRule.startSolr(createTempDir()); - solrTestRule - .newCollection("collection1") - .withConfigSet(ExternalPaths.DEFAULT_CONFIGSET) - .create(); + solrTestRule.startSolr(); + solrTestRule.newCollection().withConfigSet(ExternalPaths.DEFAULT_CONFIGSET).create(); } private GenericSolrRequest createShowFileRequest(SolrParams params) { diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/NodeHealthSolrCloudTest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/NodeHealthSolrCloudTest.java new file mode 100644 index 000000000000..61ab10b4acd3 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/NodeHealthSolrCloudTest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.admin.api; + +import static org.apache.solr.client.api.model.NodeHealthResponse.NodeStatus.OK; +import static org.hamcrest.Matchers.containsString; + +import java.util.concurrent.TimeUnit; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.NodeApi; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.embedded.JettySolrRunner; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Tests for the node-health API, on SolrCloud clusters + * + * @see NodeHealthStandaloneTest + */ +public class NodeHealthSolrCloudTest extends SolrCloudTestCase { + + @BeforeClass + public static void setupCluster() throws Exception { + configureCluster(1).addConfig("conf", configset("cloud-minimal")).configure(); + + CollectionAdminRequest.createCollection(DEFAULT_TEST_COLLECTION_NAME, "conf", 1, 1) + .process(cluster.getSolrClient()); + } + + @Test + public void testHealthyNodeReturnsOkStatus() throws Exception { + final var request = new NodeApi.Healthcheck(); + final var response = request.process(cluster.getSolrClient()); + + assertNotNull(response); + assertEquals(OK, response.status); + assertNull("Expected no error on a healthy node", response.error); + } + + @Test + public void testRequireHealthyCoresReturnOkWhenAllCoresHealthy() throws Exception { + final var request = new NodeApi.Healthcheck(); + request.setRequireHealthyCores(true); + final var response = request.process(cluster.getSolrClient()); + + assertNotNull(response); + assertEquals(OK, response.status); + assertEquals("All cores are healthy", response.message); + } + + @Test + public void testCloudMode_UnhealthyWhenZkClientClosed() throws Exception { + // Use a fresh node so closing its ZK client does not break the primary cluster node + JettySolrRunner newJetty = cluster.startJettySolrRunner(); + cluster.waitForNode(newJetty, 30); + try (SolrClient nodeClient = newJetty.newClient()) { + // Sanity check: the new node should start out healthy + assertEquals(OK, new NodeApi.Healthcheck().process(nodeClient).status); + + // Break the ZK connection to put the node into an unhealthy state + newJetty.getCoreContainer().getZkController().getZkClient().close(); + + SolrException e = + assertThrows(SolrException.class, () -> new NodeApi.Healthcheck().process(nodeClient)); + assertEquals(ErrorCode.SERVICE_UNAVAILABLE.code, e.code()); + assertThat(e.getMessage(), containsString(("Host Unavailable"))); + } finally { + newJetty.stop(); + } + } + + /** + * Verifies that when the node's name is absent from ZooKeeper's live-nodes set (while the ZK + * session itself is still connected), the v2 health-check API throws a {@code + * SERVICE_UNAVAILABLE} exception with a message identifying the live-nodes check as the cause. + * + *

This specifically exercises the code path at NodeHealth#getClusterState() that checks {@code + * clusterState.getLiveNodes().contains(nodeName)}. + */ + @Test + public void testNotInLiveNodes_ThrowsServiceUnavailable() throws Exception { + JettySolrRunner newJetty = cluster.startJettySolrRunner(); + cluster.waitForNode(newJetty, 30); + try (SolrClient nodeClient = newJetty.newClient()) { + // Sanity check: the new node should start out healthy + assertEquals(OK, new NodeApi.Healthcheck().process(nodeClient).status); + + String nodeName = newJetty.getCoreContainer().getZkController().getNodeName(); + + // Remove the node from ZooKeeper's live_nodes without closing the ZK session. + // This ensures the "ZK not connected" check passes and only the "not in live nodes" + // check fires, isolating the code path under test. + newJetty.getCoreContainer().getZkController().removeEphemeralLiveNode(); + + // Wait for the node's own ZkStateReader to reflect the removal before querying it. + newJetty + .getCoreContainer() + .getZkController() + .getZkStateReader() + .waitForLiveNodes(10, TimeUnit.SECONDS, missingLiveNode(nodeName)); + + SolrException e = + assertThrows(SolrException.class, () -> new NodeApi.Healthcheck().process(nodeClient)); + assertEquals(ErrorCode.SERVICE_UNAVAILABLE.code, e.code()); + assertThat(e.getMessage(), containsString("Not in live nodes")); + } finally { + newJetty.stop(); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/NodeHealthStandaloneTest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/NodeHealthStandaloneTest.java new file mode 100644 index 000000000000..0e3c2765038d --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/NodeHealthStandaloneTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.admin.api; + +import static org.apache.solr.client.api.model.NodeHealthResponse.NodeStatus.FAILURE; +import static org.apache.solr.client.api.model.NodeHealthResponse.NodeStatus.OK; +import static org.hamcrest.Matchers.containsString; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.request.NodeApi; +import org.apache.solr.util.SolrJettyTestRule; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +/** + * Tests for the node-health API, on Standalone Solr + * + * @see NodeHealthSolrCloudTest + */ +public class NodeHealthStandaloneTest extends SolrTestCaseJ4 { + + @ClassRule public static SolrJettyTestRule solrTestRule = new SolrJettyTestRule(); + + @BeforeClass + public static void setupCluster() throws Exception { + solrTestRule.startSolr(createTempDir()); + } + + @Test + public void testWithoutMaxGenerationLagReturnsOk() throws Exception { + + final var request = new NodeApi.Healthcheck(); + final var response = request.process(solrTestRule.getAdminClient()); + + assertNotNull(response); + assertEquals(OK, response.status); + assertThat(response.message, containsString("maxGenerationLag isn't specified")); + } + + @Test + public void testWithNegativeMaxGenerationLagReturnsFailure() throws Exception { + final var request = new NodeApi.Healthcheck(); + request.setMaxGenerationLag(-1); + final var response = request.process(solrTestRule.getAdminClient()); + + assertNotNull(response); + assertEquals(FAILURE, response.status); + assertThat(response.message, containsString("Invalid value of maxGenerationLag")); + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/RenameCoreAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/RenameCoreAPITest.java index b999e2b8cf58..d62d9bf4dd66 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/RenameCoreAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/RenameCoreAPITest.java @@ -50,11 +50,8 @@ public class RenameCoreAPITest extends SolrTestCaseJ4 { public static void beforeTest() throws Exception { EnvUtils.setProperty( ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME.toAbsolutePath().toString()); - solrTestRule.startSolr(createTempDir()); - solrTestRule - .newCollection(DEFAULT_TEST_CORENAME) - .withConfigSet(ExternalPaths.DEFAULT_CONFIGSET) - .create(); + solrTestRule.startSolr(); + solrTestRule.newCollection().withConfigSet(ExternalPaths.DEFAULT_CONFIGSET).create(); } @Test diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/V2NodeAPIMappingTest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/V2NodeAPIMappingTest.java index 18a09fc75686..6b3c63de45b4 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/V2NodeAPIMappingTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/V2NodeAPIMappingTest.java @@ -34,7 +34,6 @@ import org.apache.solr.common.util.ContentStreamBase; import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.handler.admin.CoreAdminHandler; -import org.apache.solr.handler.admin.HealthCheckHandler; import org.apache.solr.handler.admin.InfoHandler; import org.apache.solr.handler.admin.LoggingHandler; import org.apache.solr.handler.admin.PropertiesRequestHandler; @@ -55,7 +54,6 @@ public class V2NodeAPIMappingTest extends SolrTestCaseJ4 { private InfoHandler infoHandler; private LoggingHandler mockLoggingHandler; private PropertiesRequestHandler mockPropertiesHandler; - private HealthCheckHandler mockHealthCheckHandler; private ThreadDumpHandler mockThreadDumpHandler; @BeforeClass @@ -69,13 +67,11 @@ public void setupApiBag() { infoHandler = mock(InfoHandler.class); mockLoggingHandler = mock(LoggingHandler.class); mockPropertiesHandler = mock(PropertiesRequestHandler.class); - mockHealthCheckHandler = mock(HealthCheckHandler.class); mockThreadDumpHandler = mock(ThreadDumpHandler.class); queryRequestCaptor = ArgumentCaptor.forClass(SolrQueryRequest.class); when(infoHandler.getLoggingHandler()).thenReturn(mockLoggingHandler); when(infoHandler.getPropertiesHandler()).thenReturn(mockPropertiesHandler); - when(infoHandler.getHealthCheckHandler()).thenReturn(mockHealthCheckHandler); when(infoHandler.getThreadDumpHandler()).thenReturn(mockThreadDumpHandler); apiBag = new ApiBag(false); @@ -141,19 +137,6 @@ public void testThreadDumpApiAllProperties() throws Exception { assertEquals("anyParamValue", v1Params.get("anyParamName")); } - @Test - public void testHealthCheckApiAllProperties() throws Exception { - final ModifiableSolrParams solrParams = new ModifiableSolrParams(); - solrParams.add("requireHealthyCores", "true"); - solrParams.add("maxGenerationLag", "123"); - final SolrParams v1Params = - captureConvertedHealthCheckV1Params("/node/health", "GET", solrParams); - - // All parameters are passed through to v1 API as-is. - assertEquals(true, v1Params.getBool("requireHealthyCores")); - assertEquals(123, v1Params.getPrimitiveInt("maxGenerationLag")); - } - private SolrParams captureConvertedCoreV1Params(String path, String method, String v2RequestBody) throws Exception { return doCaptureParams( @@ -165,11 +148,6 @@ private SolrParams captureConvertedPropertiesV1Params( return doCaptureParams(path, method, inputParams, null, mockPropertiesHandler); } - private SolrParams captureConvertedHealthCheckV1Params( - String path, String method, SolrParams inputParams) throws Exception { - return doCaptureParams(path, method, inputParams, null, mockHealthCheckHandler); - } - private SolrParams captureConvertedThreadDumpV1Params( String path, String method, SolrParams inputParams) throws Exception { return doCaptureParams(path, method, inputParams, null, mockThreadDumpHandler); @@ -212,6 +190,5 @@ private static void registerAllNodeApis( apiBag.registerObject(new RejoinLeaderElectionAPI(coreHandler)); apiBag.registerObject(new NodePropertiesAPI(infoHandler.getPropertiesHandler())); apiBag.registerObject(new NodeThreadsAPI(infoHandler.getThreadDumpHandler())); - apiBag.registerObject(new NodeHealthAPI(infoHandler.getHealthCheckHandler())); } } diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java index 0295bd6a5eb5..5953a970cd03 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java @@ -59,7 +59,7 @@ public static void createThings() throws Exception { systemSetPropertyEnableUrlAllowList(false); EnvUtils.setProperty( ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME.toAbsolutePath().toString()); - solrTestRule.startSolr(createTempDir()); + solrTestRule.startSolr(); solrTestRule .newCollection("collection1") diff --git a/solr/core/src/test/org/apache/solr/metrics/JvmMetricsTest.java b/solr/core/src/test/org/apache/solr/metrics/JvmMetricsTest.java index b3b9f42cd5d4..f099ba7d1d3a 100644 --- a/solr/core/src/test/org/apache/solr/metrics/JvmMetricsTest.java +++ b/solr/core/src/test/org/apache/solr/metrics/JvmMetricsTest.java @@ -16,18 +16,22 @@ */ package org.apache.solr.metrics; +import com.sun.management.OperatingSystemMXBean; import io.opentelemetry.exporter.prometheus.PrometheusMetricReader; import io.prometheus.metrics.model.snapshots.MetricSnapshots; +import java.lang.management.ManagementFactory; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; import java.util.Set; import java.util.stream.Collectors; +import org.apache.lucene.util.SuppressForbidden; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.core.NodeConfig; import org.apache.solr.core.SolrXmlConfig; import org.apache.solr.util.SolrJettyTestRule; +import org.junit.Assume; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; @@ -50,7 +54,7 @@ public class JvmMetricsTest extends SolrTestCaseJ4 { @BeforeClass public static void beforeTest() throws Exception { System.setProperty("solr.metrics.jvm.enabled", "true"); - solrTestRule.startSolr(createTempDir()); + solrTestRule.startSolr(); } @Test @@ -105,4 +109,52 @@ public void testSetupJvmMetrics() throws InterruptedException { "Should have JVM buffer metrics", metricNames.stream().anyMatch(name -> name.startsWith("jvm_buffer"))); } + + @Test + @SuppressForbidden(reason = "Testing com.sun.management.OperatingSystemMXBean availability") + public void testSystemMemoryMetrics() { + PrometheusMetricReader reader = + solrTestRule + .getJetty() + .getCoreContainer() + .getMetricManager() + .getPrometheusMetricReader("solr.jvm"); + MetricSnapshots snapshots = reader.collect(); + + Set metricNames = + snapshots.stream() + .map(metric -> metric.getMetadata().getPrometheusName()) + .collect(Collectors.toSet()); + + // Physical memory metrics are only present when com.sun.management.OperatingSystemMXBean + // is available. If absent, the test is skipped. + boolean isHotSpot = + ManagementFactory.getOperatingSystemMXBean() instanceof OperatingSystemMXBean; + Assume.assumeTrue( + "Skipping: com.sun.management.OperatingSystemMXBean not available", isHotSpot); + + assertTrue( + "Should have jvm_system_memory_bytes metric (with state=total and state=free)", + metricNames.contains("jvm_system_memory_bytes")); + } + + @Test + public void testJvmMetricsDisabledNoSystemMemory() throws Exception { + // Verify that when JVM metrics are disabled, initialization is a no-op and close() is safe + SolrMetricManager metricManager = solrTestRule.getJetty().getCoreContainer().getMetricManager(); + String prevValue = System.getProperty("solr.metrics.jvm.enabled"); + System.setProperty("solr.metrics.jvm.enabled", "false"); + try { + OtelRuntimeJvmMetrics disabledMetrics = new OtelRuntimeJvmMetrics(); + OtelRuntimeJvmMetrics result = disabledMetrics.initialize(metricManager, "solr.jvm"); + assertFalse("Should not be initialized when JVM metrics disabled", result.isInitialized()); + disabledMetrics.close(); // must not throw + } finally { + if (prevValue == null) { + System.clearProperty("solr.metrics.jvm.enabled"); + } else { + System.setProperty("solr.metrics.jvm.enabled", prevValue); + } + } + } } diff --git a/solr/core/src/test/org/apache/solr/response/TestErrorResponseStackTrace.java b/solr/core/src/test/org/apache/solr/response/TestErrorResponseStackTrace.java index cda8bbf8130e..21e08e162c37 100644 --- a/solr/core/src/test/org/apache/solr/response/TestErrorResponseStackTrace.java +++ b/solr/core/src/test/org/apache/solr/response/TestErrorResponseStackTrace.java @@ -24,7 +24,6 @@ import org.apache.commons.codec.CharEncoding; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; -import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; import org.apache.solr.client.solrj.RemoteSolrException; @@ -67,7 +66,7 @@ public static void setupSolrHome() throws Exception { + " \n" + "")); - solrTestRule.startSolr(LuceneTestCase.createTempDir()); + solrTestRule.startSolr(); solrTestRule.newCollection().withConfigSet(configSet).create(); } diff --git a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java index a8250a1ba3a2..f229b01ac67a 100644 --- a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java +++ b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java @@ -26,7 +26,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.request.MetricsRequest; @@ -51,7 +50,7 @@ public class TestPrometheusResponseWriter extends SolrTestCaseJ4 { public static void beforeClass() throws Exception { EnvUtils.setProperty( ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME.toAbsolutePath().toString()); - solrTestRule.startSolr(LuceneTestCase.createTempDir()); + solrTestRule.startSolr(); solrTestRule.newCollection("core1").withConfigSet(ExternalPaths.DEFAULT_CONFIGSET).create(); solrTestRule.newCollection("core2").withConfigSet(ExternalPaths.DEFAULT_CONFIGSET).create(); diff --git a/solr/core/src/test/org/apache/solr/search/TestDocValuesIteratorCache.java b/solr/core/src/test/org/apache/solr/search/TestDocValuesIteratorCache.java index 48016588de3e..e71090e0efc2 100644 --- a/solr/core/src/test/org/apache/solr/search/TestDocValuesIteratorCache.java +++ b/solr/core/src/test/org/apache/solr/search/TestDocValuesIteratorCache.java @@ -50,7 +50,7 @@ protected void before() throws Throwable { // existence of multiple segments; if the merge policy happens to combine into a single // segment, no OrdinalMap will be built, throwing off our tests systemSetPropertySolrTestsMergePolicyFactory(NoMergePolicyFactory.class.getName()); - startSolr(LuceneTestCase.createTempDir()); + startSolr(); } }; diff --git a/solr/core/src/test/org/apache/solr/search/join/CrossCollectionJoinQueryTest.java b/solr/core/src/test/org/apache/solr/search/join/CrossCollectionJoinQueryTest.java index 91cbc3bbd8b6..70d77363539c 100644 --- a/solr/core/src/test/org/apache/solr/search/join/CrossCollectionJoinQueryTest.java +++ b/solr/core/src/test/org/apache/solr/search/join/CrossCollectionJoinQueryTest.java @@ -23,6 +23,7 @@ import java.util.Collections; import java.util.List; import java.util.Locale; +import org.apache.lucene.search.Query; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; @@ -30,8 +31,16 @@ import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.ShardParams; import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrQueryRequestBase; +import org.apache.solr.search.QueryParsing; +import org.apache.solr.util.SolrJMetricTestUtils; import org.junit.BeforeClass; import org.junit.Test; @@ -305,6 +314,295 @@ public void testAllowSolrUrlsList() throws Exception { } } + @Test + public void testShardsPreferenceRequestParamPropagation() throws Exception { + // shards.preference set as a request-level param (not in localParams) should be + // propagated to the query's otherParams + ModifiableSolrParams requestParams = new ModifiableSolrParams(); + requestParams.set(ShardParams.SHARDS_PREFERENCE, "replica.leader:false"); + + ModifiableSolrParams localParams = new ModifiableSolrParams(); + localParams.set(QueryParsing.V, "*:*"); + localParams.set(CrossCollectionJoinQParser.FROM_INDEX, "products"); + localParams.set(CrossCollectionJoinQParser.FROM, "product_id_s"); + localParams.set(CrossCollectionJoinQParser.TO, "product_id_s"); + localParams.set(CrossCollectionJoinQParser.ROUTED_BY_JOIN_KEY, "false"); + + try (SolrQueryRequest req = new SolrQueryRequestBase(null, requestParams) {}) { + CrossCollectionJoinQParser parser = + new CrossCollectionJoinQParser( + null, localParams, requestParams, req, "product_id_s", null); + Query query = parser.parse(); + + CrossCollectionJoinQuery ccjQuery = (CrossCollectionJoinQuery) query; + assertEquals("replica.leader:false", ccjQuery.otherParams.get(ShardParams.SHARDS_PREFERENCE)); + } + } + + @Test + public void testShardsPreferenceLocalParamTakesPrecedence() throws Exception { + // When shards.preference is set in both localParams and request params, + // the localParams value should take precedence + ModifiableSolrParams requestParams = new ModifiableSolrParams(); + requestParams.set(ShardParams.SHARDS_PREFERENCE, "replica.leader:false"); + + ModifiableSolrParams localParams = new ModifiableSolrParams(); + localParams.set(QueryParsing.V, "*:*"); + localParams.set(CrossCollectionJoinQParser.FROM_INDEX, "products"); + localParams.set(CrossCollectionJoinQParser.FROM, "product_id_s"); + localParams.set(CrossCollectionJoinQParser.TO, "product_id_s"); + localParams.set(CrossCollectionJoinQParser.ROUTED_BY_JOIN_KEY, "false"); + localParams.set(ShardParams.SHARDS_PREFERENCE, "replica.leader:true"); + + try (SolrQueryRequest req = new SolrQueryRequestBase(null, requestParams) {}) { + CrossCollectionJoinQParser parser = + new CrossCollectionJoinQParser( + null, localParams, requestParams, req, "product_id_s", null); + Query query = parser.parse(); + + CrossCollectionJoinQuery ccjQuery = (CrossCollectionJoinQuery) query; + // localParams value should take precedence over request-level param + assertEquals("replica.leader:true", ccjQuery.otherParams.get(ShardParams.SHARDS_PREFERENCE)); + } + } + + @Test + public void testShardsPreferenceWithCrossCollectionJoin() throws Exception { + // Use 1 shard with 2 replicas for the "from" collection so there is exactly + // 1 leader and 1 non-leader, each on a different node. This lets us verify + // via per-node /export metrics which replica actually served the stream request. + final String fromCollection = "products_pref_test"; + final String toCollection = "parts_pref_test"; + try { + CollectionAdminRequest.createCollection(fromCollection, "ccjoin", 1, 2) + .process(cluster.getSolrClient()); + CollectionAdminRequest.createCollection(toCollection, "ccjoin", 1, 1) + .process(cluster.getSolrClient()); + cluster.waitForActiveCollection(fromCollection, 1, 2); + cluster.waitForActiveCollection(toCollection, 1, 1); + + // Index test data + List productDocs = new ArrayList<>(); + List partDocs = new ArrayList<>(); + for (int productId = 0; productId < NUM_PRODUCTS; ++productId) { + int sizeNum = productId % SIZES.length; + String size = SIZES[sizeNum]; + productDocs.add( + new SolrInputDocument( + "id", String.valueOf(productId), + "product_id_s", String.valueOf(productId), + "size_s", size)); + for (int partNum = 0; partNum <= sizeNum; partNum++) { + String partId = String.format(Locale.ROOT, "%d_%d", productId, partNum); + partDocs.add( + new SolrInputDocument("id", partId, "product_id_s", String.valueOf(productId))); + } + } + indexDocs(fromCollection, productDocs); + cluster.getSolrClient().commit(fromCollection); + indexDocs(toCollection, partDocs); + cluster.getSolrClient().commit(toCollection); + + // Identify leader and non-leader replicas for the "from" collection's single shard + DocCollection fromDocCollection = + cluster.getSolrClient().getClusterState().getCollection(fromCollection); + Slice shard = fromDocCollection.getSlices().iterator().next(); + Replica leader = shard.getLeader(); + assertNotNull("Leader should exist for shard", leader); + Replica nonLeader = + shard.getReplicas().stream() + .filter(r -> !r.getName().equals(leader.getName())) + .findFirst() + .orElseThrow(() -> new AssertionError("Expected a non-leader replica")); + + String leaderBaseUrl = leader.getBaseUrl(); + String nonLeaderBaseUrl = nonLeader.getBaseUrl(); + assertNotEquals( + "Leader and non-leader should be on different nodes for this test to be meaningful", + leaderBaseUrl, + nonLeaderBaseUrl); + + // --- Test 1: replica.leader:false should route /export to the non-leader --- + double leaderCountBefore = getNumExportRequests(leaderBaseUrl, fromCollection); + double nonLeaderCountBefore = getNumExportRequests(nonLeaderBaseUrl, fromCollection); + + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set( + "q", + String.format( + Locale.ROOT, + "{!join method=crossCollection fromIndex=%s from=product_id_s to=product_id_s routed=false ttl=0}size_s:M", + fromCollection)); + params.set("rows", "0"); + params.set(ShardParams.SHARDS_PREFERENCE, "replica.leader:false"); + + QueryResponse resp = cluster.getSolrClient().query(toCollection, params); + assertEquals(NUM_PRODUCTS / 2, resp.getResults().getNumFound()); + + double leaderCountAfter = getNumExportRequests(leaderBaseUrl, fromCollection); + double nonLeaderCountAfter = getNumExportRequests(nonLeaderBaseUrl, fromCollection); + + assertTrue( + "Non-leader replica should have received the /export request" + + " (before=" + + nonLeaderCountBefore + + ", after=" + + nonLeaderCountAfter + + ")", + nonLeaderCountAfter > nonLeaderCountBefore); + assertEquals( + "Leader replica should NOT have received the /export request", + leaderCountBefore, + leaderCountAfter, + 0.0); + + // --- Test 2: replica.leader:true should route /export to the leader --- + leaderCountBefore = leaderCountAfter; + nonLeaderCountBefore = nonLeaderCountAfter; + + params.set(ShardParams.SHARDS_PREFERENCE, "replica.leader:true"); + resp = cluster.getSolrClient().query(toCollection, params); + assertEquals(NUM_PRODUCTS / 2, resp.getResults().getNumFound()); + + leaderCountAfter = getNumExportRequests(leaderBaseUrl, fromCollection); + nonLeaderCountAfter = getNumExportRequests(nonLeaderBaseUrl, fromCollection); + + assertTrue( + "Leader replica should have received the /export request" + + " (before=" + + leaderCountBefore + + ", after=" + + leaderCountAfter + + ")", + leaderCountAfter > leaderCountBefore); + assertEquals( + "Non-leader replica should NOT have received the /export request", + nonLeaderCountBefore, + nonLeaderCountAfter, + 0.0); + } finally { + CollectionAdminRequest.deleteCollection(toCollection).process(cluster.getSolrClient()); + CollectionAdminRequest.deleteCollection(fromCollection).process(cluster.getSolrClient()); + } + } + + @Test + public void testShardsPreferenceLocationLocal() throws Exception { + // Test that replica.location:local routes the join's /export stream to a replica + // on the same node where the join query is processed. This validates that the + // RequestReplicaListTransformerGenerator is initialized with full node context + // (nodeName, baseUrl, hostName). + final String fromCollection = "products_local_test"; + final String toCollection = "parts_local_test"; + try { + // "from" collection: 1 shard, NUM_NODES replicas → one replica on every node + CollectionAdminRequest.createCollection(fromCollection, "ccjoin", 1, NUM_NODES) + .process(cluster.getSolrClient()); + // "to" collection: 1 shard, 1 replica → on exactly one node + CollectionAdminRequest.createCollection(toCollection, "ccjoin", 1, 1) + .process(cluster.getSolrClient()); + cluster.waitForActiveCollection(fromCollection, 1, NUM_NODES); + cluster.waitForActiveCollection(toCollection, 1, 1); + + // Index test data + List productDocs = new ArrayList<>(); + List partDocs = new ArrayList<>(); + for (int productId = 0; productId < NUM_PRODUCTS; ++productId) { + int sizeNum = productId % SIZES.length; + String size = SIZES[sizeNum]; + productDocs.add( + new SolrInputDocument( + "id", String.valueOf(productId), + "product_id_s", String.valueOf(productId), + "size_s", size)); + for (int partNum = 0; partNum <= sizeNum; partNum++) { + String partId = String.format(Locale.ROOT, "%d_%d", productId, partNum); + partDocs.add( + new SolrInputDocument("id", partId, "product_id_s", String.valueOf(productId))); + } + } + indexDocs(fromCollection, productDocs); + cluster.getSolrClient().commit(fromCollection); + indexDocs(toCollection, partDocs); + cluster.getSolrClient().commit(toCollection); + + // Find the node hosting the "to" collection's shard. The join stream will execute + // on this node, so replica.location:local should prefer the "from" replica here. + DocCollection toDocCollection = + cluster.getSolrClient().getClusterState().getCollection(toCollection); + Slice toShard = toDocCollection.getSlices().iterator().next(); + String toNodeBaseUrl = toShard.getReplicas().iterator().next().getBaseUrl(); + + // Collect all "from" replica base URLs + DocCollection fromDocCollection = + cluster.getSolrClient().getClusterState().getCollection(fromCollection); + Slice fromShard = fromDocCollection.getSlices().iterator().next(); + List fromBaseUrls = new ArrayList<>(); + for (Replica r : fromShard.getReplicas()) { + fromBaseUrls.add(r.getBaseUrl()); + } + assertTrue( + "The 'from' collection should have a replica on the same node as the 'to' collection", + fromBaseUrls.contains(toNodeBaseUrl)); + + // Get baseline /export request counts for the "from" collection on all nodes + double localCountBefore = getNumExportRequests(toNodeBaseUrl, fromCollection); + List remoteCountsBefore = new ArrayList<>(); + for (String baseUrl : fromBaseUrls) { + if (!baseUrl.equals(toNodeBaseUrl)) { + remoteCountsBefore.add(new double[] {getNumExportRequests(baseUrl, fromCollection)}); + } + } + + // Execute join query with replica.location:local + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set( + "q", + String.format( + Locale.ROOT, + "{!join method=crossCollection fromIndex=%s from=product_id_s to=product_id_s routed=false ttl=0}size_s:M", + fromCollection)); + params.set("rows", "0"); + params.set(ShardParams.SHARDS_PREFERENCE, "replica.location:local"); + + QueryResponse resp = cluster.getSolrClient().query(toCollection, params); + assertEquals(NUM_PRODUCTS / 2, resp.getResults().getNumFound()); + + // Verify the local node's "from" replica received the /export request + double localCountAfter = getNumExportRequests(toNodeBaseUrl, fromCollection); + assertTrue( + "Local 'from' replica should have received the /export request" + + " (before=" + + localCountBefore + + ", after=" + + localCountAfter + + ")", + localCountAfter > localCountBefore); + + // Verify remote nodes did NOT receive /export requests + int remoteIdx = 0; + for (String baseUrl : fromBaseUrls) { + if (!baseUrl.equals(toNodeBaseUrl)) { + double remoteCountAfter = getNumExportRequests(baseUrl, fromCollection); + assertEquals( + "Remote 'from' replica on " + baseUrl + " should NOT have received /export request", + remoteCountsBefore.get(remoteIdx)[0], + remoteCountAfter, + 0.0); + remoteIdx++; + } + } + } finally { + CollectionAdminRequest.deleteCollection(toCollection).process(cluster.getSolrClient()); + CollectionAdminRequest.deleteCollection(fromCollection).process(cluster.getSolrClient()); + } + } + + private static double getNumExportRequests(String baseUrl, String collectionName) + throws SolrServerException, IOException { + return SolrJMetricTestUtils.getNumCoreRequests(baseUrl, collectionName, "QUERY", "/export"); + } + public void testCcJoinQuery(String query, boolean expectFullResults) throws Exception { assertResultCount("parts", query, NUM_PRODUCTS / 2, expectFullResults); } diff --git a/solr/core/src/test/org/apache/solr/search/json/TestJsonRequestWithEdismaxDefType.java b/solr/core/src/test/org/apache/solr/search/json/TestJsonRequestWithEdismaxDefType.java index 1339d1d64b2c..806c57f72c87 100644 --- a/solr/core/src/test/org/apache/solr/search/json/TestJsonRequestWithEdismaxDefType.java +++ b/solr/core/src/test/org/apache/solr/search/json/TestJsonRequestWithEdismaxDefType.java @@ -32,7 +32,7 @@ public class TestJsonRequestWithEdismaxDefType extends SolrTestCaseJ4 { @ClassRule public static final SolrClientTestRule solrTestRule = new EmbeddedSolrServerTestRule(); public void test() throws Exception { - solrTestRule.startSolr(LuceneTestCase.createTempDir()); + solrTestRule.startSolr(); Path configSet = LuceneTestCase.createTempDir(); SolrTestCaseJ4.copyMinConf(configSet); diff --git a/solr/core/src/test/org/apache/solr/servlet/HideStackTraceTest.java b/solr/core/src/test/org/apache/solr/servlet/HideStackTraceTest.java index 2caca8ee68b0..0a0986292be0 100644 --- a/solr/core/src/test/org/apache/solr/servlet/HideStackTraceTest.java +++ b/solr/core/src/test/org/apache/solr/servlet/HideStackTraceTest.java @@ -24,7 +24,6 @@ import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.util.EntityUtils; -import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; import org.apache.solr.client.solrj.apache.HttpClientUtil; @@ -66,7 +65,7 @@ public static void setupSolrHome() throws Exception { + " \n" + "")); - solrTestRule.startSolr(LuceneTestCase.createTempDir()); + solrTestRule.startSolr(); solrTestRule.newCollection().withConfigSet(configSet).create(); } diff --git a/solr/core/src/test/org/apache/solr/update/CustomTLogDirTest.java b/solr/core/src/test/org/apache/solr/update/CustomTLogDirTest.java index 03bc05039246..334b8c3525fb 100644 --- a/solr/core/src/test/org/apache/solr/update/CustomTLogDirTest.java +++ b/solr/core/src/test/org/apache/solr/update/CustomTLogDirTest.java @@ -39,7 +39,7 @@ public class CustomTLogDirTest extends SolrTestCaseJ4 { @Override protected void before() { System.setProperty("solr.directoryFactory", "solr.NRTCachingDirectoryFactory"); - solrTestRule.startSolr(LuceneTestCase.createTempDir()); + solrTestRule.startSolr(); } }; diff --git a/solr/core/src/test/org/apache/solr/update/RootFieldTest.java b/solr/core/src/test/org/apache/solr/update/RootFieldTest.java index d5ca7c660766..1487e9ec9dd6 100644 --- a/solr/core/src/test/org/apache/solr/update/RootFieldTest.java +++ b/solr/core/src/test/org/apache/solr/update/RootFieldTest.java @@ -61,7 +61,11 @@ public static void beforeTest() throws Exception { String schema = useRootSchema ? "schema15.xml" : "schema-rest.xml"; SolrTestCaseJ4.newRandomConfig(); - solrTestRule.newCollection().withConfigSet("../collection1").withSchemaFile(schema).create(); + solrTestRule + .newCollection() + .withConfigSet(SolrTestCaseJ4.TEST_COLL1_CONF()) + .withSchemaFile(schema) + .create(); } @Test diff --git a/solr/core/src/test/org/apache/solr/update/processor/AbstractAtomicUpdatesMultivalueTestBase.java b/solr/core/src/test/org/apache/solr/update/processor/AbstractAtomicUpdatesMultivalueTestBase.java index dfd1ac8980e5..2852df097ece 100644 --- a/solr/core/src/test/org/apache/solr/update/processor/AbstractAtomicUpdatesMultivalueTestBase.java +++ b/solr/core/src/test/org/apache/solr/update/processor/AbstractAtomicUpdatesMultivalueTestBase.java @@ -16,6 +16,7 @@ */ package org.apache.solr.update.processor; +import static org.apache.solr.SolrTestCaseJ4.TEST_COLL1_CONF; import static org.apache.solr.SolrTestCaseJ4.sdoc; import static org.hamcrest.CoreMatchers.hasItems; import static org.hamcrest.CoreMatchers.not; @@ -55,7 +56,7 @@ protected static void initWithRequestWriter(RequestWriterSupplier requestWriterS System.setProperty("solr.index.updatelog.enabled", "true"); SolrTestCaseJ4.newRandomConfig(); - solrTestRule.newCollection().withConfigSet("../collection1").create(); + solrTestRule.newCollection().withConfigSet(TEST_COLL1_CONF()).create(); } @Before diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java index 2c0a1684ec8d..23470049c985 100644 --- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java +++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java @@ -38,10 +38,8 @@ public class ExtractingRequestHandlerTikaServerTest extends ExtractingRequestHan @BeforeClass @SuppressWarnings("resource") public static void beforeClassTika() { - // skip this test on s390x Assume.assumeFalse( - "Skipping ExtractingRequestHandlerTikaServerTest on s390x", - "s390x".equalsIgnoreCase(System.getProperty("os.arch"))); + "Skipping on s390x", "s390x".equalsIgnoreCase(System.getProperty("os.arch"))); String baseUrl; try { diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java index 49dca514e248..b7723da94ff9 100644 --- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java +++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java @@ -66,6 +66,9 @@ public boolean reject(Thread t) { @SuppressWarnings("resource") @BeforeClass public static void startTikaServer() { + Assume.assumeFalse( + "Skipping on s390x", "s390x".equalsIgnoreCase(System.getProperty("os.arch"))); + try { tika = new GenericContainer<>("apache/tika:3.2.3.0-full").withExposedPorts(9998); tika.start(); diff --git a/solr/modules/sql/src/test/org/apache/solr/handler/sql/TestSQLHandlerNonCloud.java b/solr/modules/sql/src/test/org/apache/solr/handler/sql/TestSQLHandlerNonCloud.java index 8bb82fd41b05..cae84b3a3a90 100644 --- a/solr/modules/sql/src/test/org/apache/solr/handler/sql/TestSQLHandlerNonCloud.java +++ b/solr/modules/sql/src/test/org/apache/solr/handler/sql/TestSQLHandlerNonCloud.java @@ -47,8 +47,7 @@ private static Path createSolrHome() throws Exception { @BeforeClass public static void beforeClass() throws Exception { - Path solrHome = createSolrHome(); - solrTestRule.startSolr(solrHome); + solrTestRule.startSolr(createSolrHome()); } @Test diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/implicit-requesthandlers.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/implicit-requesthandlers.adoc index 16b2f691281e..4380337752c9 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/implicit-requesthandlers.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/implicit-requesthandlers.adoc @@ -40,18 +40,24 @@ This handler must have a collection name in the path to the endpoint. |`solr//admin/file` |{solr-javadocs}/core/org/apache/solr/handler/admin/ShowFileRequestHandler.html[ShowFileRequestHandler] |`_ADMIN_FILE` |=== -Health:: Report the health of the node (_available only in SolrCloud mode_) +Health:: Report the health of the node. + [cols="3*.",frame=none,grid=cols,options="header"] |=== |API Endpoints |Class & Javadocs |Paramset |v1: `solr/admin/info/health` -v2: `api/node/health` |{solr-javadocs}/core/org/apache/solr/handler/admin/HealthCheckHandler.html[HealthCheckHandler] | +v2: `api/node/health` |v1: {solr-javadocs}/core/org/apache/solr/handler/admin/HealthCheckHandler.html[HealthCheckHandler] + +v2: {solr-javadocs}/core/org/apache/solr/handler/admin/api/NodeHealth.html[NodeHealth] | |=== + -This endpoint also accepts additional request parameters. -Please see {solr-javadocs}/core/org/apache/solr/handler/admin/HealthCheckHandler.html[Javadocs] for details. +In SolrCloud mode the handler checks that the node is connected to ZooKeeper and is listed in live nodes. +The optional `requireHealthyCores=true` parameter additionally requires that all local replicas be in an active state, which is useful for rolling-restart probes. ++ +In user-managed (leader-follower) mode the handler checks replication lag. +The optional `maxGenerationLag=` parameter specifies the maximum number of Lucene commit generations by which a follower is allowed to trail its leader; the endpoint returns HTTP 503 if any core exceeds this threshold. +See xref:deployment-guide:user-managed-index-replication.adoc#monitoring-follower-replication-lag[Monitoring Follower Replication Lag] for details and examples. Logging:: Retrieve and modify registered loggers. + diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/cluster-node-management.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/cluster-node-management.adoc index c5f9b58c9ac0..46517de459d0 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/cluster-node-management.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/cluster-node-management.adoc @@ -667,7 +667,7 @@ Examining the clusterstate after issuing this call should show exactly one repli == Migrate Replicas Migrate all replicas off of a given set of source nodes. -+ + If more than one node is used as a targetNode (either explicitly, or by default), then the configured xref:configuration-guide:replica-placement-plugins.adoc[Replica Placement Plugin] will be used to determine which targetNode should be used for each migrated replica. @@ -862,7 +862,12 @@ So don't perform other collection operations in this period. == DELETENODE: Delete Replicas in a Node Deletes all replicas of all collections in that node. -Please note that the node itself will remain as a live node after this operation. +Please note that the node itself will remain as a live node after this operation if it is currently functioning. + +[TIP] +==== +If the node is currently down and you aren't bringing it back, then this command removes that node's replica entries from the cluster state (ZooKeeper), cleaning up references to its replicas. It does not change the node's liveness in `/live_nodes`, and the node may still rejoin the cluster if it is started again later. +==== [tabs#deletenode-request] ====== diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/metrics-reporting.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/metrics-reporting.adoc index 7d0e79a2c0a5..58bf50ebf022 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/metrics-reporting.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/metrics-reporting.adoc @@ -98,6 +98,21 @@ The `JVM Registry` gathers metrics from the JVM using the OpenTelemetry instrume JVM metrics are enabled by default but can be disabled by setting either the system property `-Dsolr.metrics.jvm.enabled=false` or the environment variable `SOLR_METRICS_JVM_ENABLED=false`. +==== Physical Memory Metrics + +Solr exposes a gauge for host or container physical memory, registered under the `solr.jvm` registry: + +[cols="2,1,3",options="header"] +|=== +| Prometheus Metric Name | Type | Description +| `jvm_system_memory_bytes{state="total"}` | gauge | Total physical memory of the host or container in bytes. On Linux with cgroup memory limits, reflects the container limit rather than host RAM. +| `jvm_system_memory_bytes{state="free"}` | gauge | Free (unused) physical memory of the host or container in bytes. +|=== + +NOTE: These metrics are available when the JVM provides the `com.sun.management.OperatingSystemMXBean` interface (this includes most HotSpot-derived JVMs). On JVMs that do not provide `com.sun.management.OperatingSystemMXBean`, the metrics are silently absent. + +NOTE: On Linux containers with cgroup v1 or v2 memory limits set, the JDK reports the container memory limit as the total, not the host's physical RAM. This is the correct value for calculating MMap cache efficiency in containerised deployments. + === Overseer Registry The `Overseer Registry` is initialized when running in SolrCloud mode and includes the following information: diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/user-managed-index-replication.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/user-managed-index-replication.adoc index ea3f0f376747..ff3e4421fbb5 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/user-managed-index-replication.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/user-managed-index-replication.adoc @@ -575,6 +575,63 @@ A snapshot with the name `snapshot._name_` must exist or an error will be return `location`::: The location where the snapshot is created. +[[monitoring-follower-replication-lag]] +== Monitoring Follower Replication Lag + +In a leader-follower deployment it is important to know whether followers are keeping pace with the leader. +Solr's health-check endpoint supports a `maxGenerationLag` request parameter that lets you assert that each follower core is within a specified number of Lucene commit generations of its leader. +When the follower is lagging more than the allowed number of generations the endpoint returns HTTP 503 (Service Unavailable), making it straightforward to integrate into load-balancer health probes or monitoring systems. + +The `maxGenerationLag` parameter is an integer representing the maximum acceptable number of commit generations by which a follower is allowed to trail its leader. +A value of `0` requires the follower to be fully up to date. +If the parameter is omitted, the health check returns `OK` regardless of replication lag. + +[WARNING] +==== +Because a follower's generation can only increase when a replication from the leader actually completes, `maxGenerationLag=0` may return `FAILURE` immediately after a follower starts or after a period of network instability even though the follower will catch up on the next poll cycle. +Use a small positive value (for example `2`) for production monitoring unless you require strict freshness guarantees. +==== + +Use the health endpoint as follows: + +==== +[.tab-label]*V1 API* + +[source,bash] +---- +http://_follower_host:port_/solr/admin/info/health?maxGenerationLag=<_max_lag_> +---- +==== + +==== +[.tab-label]*V2 API* + +[source,bash] +---- +http://_follower_host:port_/api/node/health?maxGenerationLag=<_max_lag_> +---- +==== + +A healthy response looks like: + +[source,json] +---- +{ + "status": "OK", + "message": "All the followers are in sync with leader (within maxGenerationLag: 2) or the cores are acting as leader" +} +---- + +When a follower is lagging too far behind, the response is HTTP 503 and the body identifies the lagging cores: + +[source,json] +---- +{ + "status": "FAILURE", + "message": "Cores violating maxGenerationLag:2.\nCore collection1 is lagging by 5 generations" +} +---- + == Optimizing Distributed Indexes Optimizing an index is not something most users should generally worry about - but in particular users should be aware of the impacts of optimizing an index when using the `ReplicationHandler`. diff --git a/solr/solrj-jetty/src/test/org/apache/solr/client/solrj/jetty/ConcurrentUpdateJettySolrClientBadInputTest.java b/solr/solrj-jetty/src/test/org/apache/solr/client/solrj/jetty/ConcurrentUpdateJettySolrClientBadInputTest.java index 1c78cbc6463a..a01a522f02cf 100644 --- a/solr/solrj-jetty/src/test/org/apache/solr/client/solrj/jetty/ConcurrentUpdateJettySolrClientBadInputTest.java +++ b/solr/solrj-jetty/src/test/org/apache/solr/client/solrj/jetty/ConcurrentUpdateJettySolrClientBadInputTest.java @@ -43,11 +43,8 @@ public class ConcurrentUpdateJettySolrClientBadInputTest extends SolrTestCaseJ4 public static void beforeTest() throws Exception { EnvUtils.setProperty( ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME.toAbsolutePath().toString()); - solrTestRule.startSolr(createTempDir()); - solrTestRule - .newCollection(DEFAULT_TEST_COLLECTION_NAME) - .withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET) - .create(); + solrTestRule.startSolr(); + solrTestRule.newCollection().withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET).create(); } @Test diff --git a/solr/solrj-jetty/src/test/org/apache/solr/client/solrj/jetty/HttpJettySolrClientCompatibilityTest.java b/solr/solrj-jetty/src/test/org/apache/solr/client/solrj/jetty/HttpJettySolrClientCompatibilityTest.java index b5aafdee9ba9..82b3fd8e83ae 100644 --- a/solr/solrj-jetty/src/test/org/apache/solr/client/solrj/jetty/HttpJettySolrClientCompatibilityTest.java +++ b/solr/solrj-jetty/src/test/org/apache/solr/client/solrj/jetty/HttpJettySolrClientCompatibilityTest.java @@ -63,10 +63,7 @@ public void testConnectToOldNodesUsingHttp1() throws Exception { EnvUtils.setProperty( ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME.toAbsolutePath().toString()); solrTestRule.startSolr(createTempDir(), new Properties(), jettyConfig); - solrTestRule - .newCollection(DEFAULT_TEST_COLLECTION_NAME) - .withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET) - .create(); + solrTestRule.newCollection().withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET).create(); try (var client = new HttpJettySolrClient.Builder(solrTestRule.getBaseUrl() + "/debug/foo") @@ -92,10 +89,7 @@ public void testConnectToNewNodesUsingHttp1() throws Exception { EnvUtils.setProperty( ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME.toAbsolutePath().toString()); solrTestRule.startSolr(createTempDir(), new Properties(), jettyConfig); - solrTestRule - .newCollection(DEFAULT_TEST_COLLECTION_NAME) - .withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET) - .create(); + solrTestRule.newCollection().withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET).create(); try (var client = new HttpJettySolrClient.Builder(solrTestRule.getBaseUrl() + "/debug/foo") @@ -125,10 +119,7 @@ public void testConnectToOldNodesUsingHttp2() throws Exception { EnvUtils.setProperty( ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME.toAbsolutePath().toString()); solrTestRule.startSolr(createTempDir(), new Properties(), jettyConfig); - solrTestRule - .newCollection(DEFAULT_TEST_COLLECTION_NAME) - .withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET) - .create(); + solrTestRule.newCollection().withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET).create(); System.clearProperty("solr.http1"); try (var client = diff --git a/solr/solrj-jetty/src/test/org/apache/solr/client/solrj/jetty/HttpJettySolrClientProxyTest.java b/solr/solrj-jetty/src/test/org/apache/solr/client/solrj/jetty/HttpJettySolrClientProxyTest.java index 20d44bc107da..7b4f85ba7fe3 100644 --- a/solr/solrj-jetty/src/test/org/apache/solr/client/solrj/jetty/HttpJettySolrClientProxyTest.java +++ b/solr/solrj-jetty/src/test/org/apache/solr/client/solrj/jetty/HttpJettySolrClientProxyTest.java @@ -51,7 +51,7 @@ public static void beforeTest() throws Exception { ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME.toAbsolutePath().toString()); // Needed for configset location solrTestRule.enableProxy(); - solrTestRule.startSolr(createTempDir()); + solrTestRule.startSolr(); // Actually only need extremely minimal configSet but just use the default solrTestRule .newCollection() diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java index 164ae8ae7b03..86c96944ace5 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java @@ -845,6 +845,12 @@ public static void reflectWrite(MapWriter.EntryWriter ew, Object o) { * @return a serializable version of the object */ public static Object getReflectWriter(Object o) { + // Enums serialized as their declared name so that javabin/NamedList consumers + // (e.g. HealthCheckHandlerTest comparing against CommonParams.OK == "OK") see + // a plain string rather than "pkg.EnumClass:NAME". + if (o instanceof Enum e) { + return e.name(); + } List fieldWriters = null; try { fieldWriters = diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java index 4f7c4029e125..4b0a1f286bed 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java @@ -103,11 +103,8 @@ public abstract class SolrExampleTests extends SolrExampleTestsBase { public static void beforeTest() throws Exception { EnvUtils.setProperty( ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME.toAbsolutePath().toString()); - solrTestRule.startSolr(createTempDir()); - solrTestRule - .newCollection(DEFAULT_TEST_COLLECTION_NAME) - .withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET) - .create(); + solrTestRule.startSolr(); + solrTestRule.newCollection().withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET).create(); } @Test diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/TestBatchUpdate.java b/solr/solrj/src/test/org/apache/solr/client/solrj/TestBatchUpdate.java index 85dd1c8266dc..d64b999560b9 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/TestBatchUpdate.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/TestBatchUpdate.java @@ -51,11 +51,8 @@ public static void beforeTest() throws Exception { EnvUtils.setProperty( ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME.toAbsolutePath().toString()); // Needed for configset location - solrTestRule.startSolr(createTempDir()); - solrTestRule - .newCollection("collection1") - .withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET) - .create(); + solrTestRule.startSolr(); + solrTestRule.newCollection().withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET).create(); } static final int numdocs = 1000; diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/TestSolrJErrorHandling.java b/solr/solrj/src/test/org/apache/solr/client/solrj/TestSolrJErrorHandling.java index 2a5ca071267c..023874d77ddc 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/TestSolrJErrorHandling.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/TestSolrJErrorHandling.java @@ -66,11 +66,8 @@ public static void beforeTest() throws Exception { EnvUtils.setProperty( ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME.toAbsolutePath().toString()); // Needed for configset location - solrTestRule.startSolr(createTempDir()); - solrTestRule - .newCollection("collection1") - .withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET) - .create(); + solrTestRule.startSolr(); + solrTestRule.newCollection().withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET).create(); } @Override diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientTestBase.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientTestBase.java index 4c2eca633c94..3ccb0f390118 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientTestBase.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientTestBase.java @@ -199,10 +199,7 @@ public static void beforeTest() throws Exception { EnvUtils.setProperty( ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME.toAbsolutePath().toString()); solrTestRule.startSolr(createTempDir(), new Properties(), jettyConfig); - solrTestRule - .newCollection(DEFAULT_TEST_COLLECTION_NAME) - .withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET) - .create(); + solrTestRule.newCollection().withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET).create(); } @AfterClass diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientBadInputTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientBadInputTest.java index 8eaa9feb82ae..c80338e412b1 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientBadInputTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientBadInputTest.java @@ -45,11 +45,8 @@ public static void beforeTest() throws Exception { EnvUtils.setProperty( ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME.toAbsolutePath().toString()); // Needed for configset location - solrTestRule.startSolr(createTempDir()); - solrTestRule - .newCollection("collection1") - .withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET) - .create(); + solrTestRule.startSolr(); + solrTestRule.newCollection().withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET).create(); } @Test diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientTestBase.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientTestBase.java index 73d6d19ccfe2..b9d76b80133c 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientTestBase.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientTestBase.java @@ -97,10 +97,7 @@ public static void beforeTest() throws Exception { EnvUtils.setProperty( ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME.toAbsolutePath().toString()); solrTestRule.startSolr(createTempDir(), new Properties(), jettyConfig); - solrTestRule - .newCollection(DEFAULT_TEST_COLLECTION_NAME) - .withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET) - .create(); + solrTestRule.newCollection().withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET).create(); } @Override diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/LB2SolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/LB2SolrClientTest.java index 66e806e95381..849d8953c0fa 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/LB2SolrClientTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/LB2SolrClientTest.java @@ -16,6 +16,7 @@ */ package org.apache.solr.client.solrj.impl; +import java.io.Closeable; import java.io.IOException; import java.io.UncheckedIOException; import java.net.ServerSocket; @@ -100,11 +101,7 @@ private void addDocs(SolrInstance solrInstance) throws IOException, SolrServerEx @Override public void tearDown() throws Exception { - for (SolrInstance aSolr : solr) { - if (aSolr != null) { - aSolr.tearDown(); - } - } + IOUtils.close(solr); // closes all solr instances in 'solr[]', and throws if failed super.tearDown(); } @@ -286,7 +283,7 @@ private void startJettyAndWaitForAliveCheckQuery(SolrInstance solrInstance) thro } } - private static class SolrInstance { + private static class SolrInstance implements Closeable { String name; Path homeDir; Path dataDir; @@ -339,9 +336,16 @@ public void setUp() throws Exception { Files.createFile(homeDir.resolve("collection1/core.properties")); } - public void tearDown() throws Exception { - if (jetty != null) jetty.stop(); - IOUtils.rm(homeDir); + @Override + public void close() throws IOException { + try { + if (jetty != null) jetty.stop(); + IOUtils.rm(homeDir); + } catch (IOException | RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException(e); + } } public void startJetty() throws Exception { diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/LBHttpSolrClientBadInputTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/LBHttpSolrClientBadInputTest.java index 647f912b11d2..e82dcb1ca9fb 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/LBHttpSolrClientBadInputTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/LBHttpSolrClientBadInputTest.java @@ -44,11 +44,8 @@ public static void beforeTest() throws Exception { EnvUtils.setProperty( ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME.toAbsolutePath().toString()); // Needed for configset location - solrTestRule.startSolr(createTempDir()); - solrTestRule - .newCollection("collection1") - .withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET) - .create(); + solrTestRule.startSolr(); + solrTestRule.newCollection().withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET).create(); } @Test diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/request/SolrPingTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/request/SolrPingTest.java index c1efae5ef565..721a94925390 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/request/SolrPingTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/request/SolrPingTest.java @@ -16,6 +16,7 @@ */ package org.apache.solr.client.solrj.request; +import java.nio.file.Path; import org.apache.solr.SolrTestCase; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.response.SolrPingResponse; @@ -35,10 +36,11 @@ public class SolrPingTest extends SolrTestCase { @BeforeClass public static void beforeClass() throws Exception { - solrTestRule.startSolr(SolrTestCaseJ4.getFile("solrj/solr")); + Path solrHome = SolrTestCaseJ4.getFile("solrj/solr"); + solrTestRule.startSolr(solrHome); SolrTestCaseJ4.newRandomConfig(); - solrTestRule.newCollection().withConfigSet("../collection1").create(); + solrTestRule.newCollection().withConfigSet(solrHome.resolve("collection1")).create(); } @Before diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/request/json/DirectJsonQueryRequestFacetingEmbeddedTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/request/json/DirectJsonQueryRequestFacetingEmbeddedTest.java index 488dd83f41cd..0717ab90c7dd 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/request/json/DirectJsonQueryRequestFacetingEmbeddedTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/request/json/DirectJsonQueryRequestFacetingEmbeddedTest.java @@ -20,7 +20,6 @@ import static org.apache.solr.SolrTestCaseJ4.getFile; import java.util.List; -import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.solr.SolrTestCase; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; import org.apache.solr.client.solrj.SolrClient; @@ -43,7 +42,6 @@ public class DirectJsonQueryRequestFacetingEmbeddedTest extends SolrTestCase { @ClassRule public static final EmbeddedSolrServerTestRule solrTestRule = new EmbeddedSolrServerTestRule(); - private static final String COLLECTION_NAME = "collection1"; private static final int NUM_TECHPRODUCTS_DOCS = 32; private static final int NUM_IN_STOCK = 17; private static final int NUM_ELECTRONICS = 12; @@ -56,17 +54,14 @@ public class DirectJsonQueryRequestFacetingEmbeddedTest extends SolrTestCase { @BeforeClass public static void beforeClass() throws Exception { - solrTestRule.startSolr(LuceneTestCase.createTempDir()); + solrTestRule.startSolr(); - solrTestRule - .newCollection(COLLECTION_NAME) - .withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET) - .create(); + solrTestRule.newCollection().withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET).create(); - SolrClient client = solrTestRule.getSolrClient(COLLECTION_NAME); + SolrClient client = solrTestRule.getSolrClient(); ContentStreamUpdateRequest up = new ContentStreamUpdateRequest("/update"); - up.setParam("collection", COLLECTION_NAME); + up.setParam("collection", client.getDefaultCollection()); up.addFile(getFile("solrj/techproducts.xml"), "application/xml"); up.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true); UpdateResponse updateResponse = up.process(client); @@ -90,7 +85,7 @@ public void testSingleTermsFacet() throws Exception { "}"); final DirectJsonQueryRequest request = new DirectJsonQueryRequest(jsonBody); - QueryResponse response = request.process(solrTestRule.getSolrClient(), COLLECTION_NAME); + QueryResponse response = request.process(solrTestRule.getSolrClient()); assertExpectedDocumentsFoundAndReturned(response, NUM_TECHPRODUCTS_DOCS, 10); final NestableJsonFacet topLevelFacetData = response.getJsonFacetingResponse(); @@ -125,7 +120,7 @@ public void testMultiTermsFacet() throws Exception { "}"); final DirectJsonQueryRequest request = new DirectJsonQueryRequest(jsonBody); - QueryResponse response = request.process(solrTestRule.getSolrClient(), COLLECTION_NAME); + QueryResponse response = request.process(solrTestRule.getSolrClient()); assertExpectedDocumentsFoundAndReturned(response, NUM_TECHPRODUCTS_DOCS, 10); final NestableJsonFacet topLevelFacetData = response.getJsonFacetingResponse(); @@ -163,7 +158,7 @@ public void testSingleRangeFacet() throws Exception { "}"); final DirectJsonQueryRequest request = new DirectJsonQueryRequest(jsonBody); - QueryResponse response = request.process(solrTestRule.getSolrClient(), COLLECTION_NAME); + QueryResponse response = request.process(solrTestRule.getSolrClient()); assertExpectedDocumentsFoundAndReturned(response, NUM_TECHPRODUCTS_DOCS, 10); final NestableJsonFacet topLevelFacetData = response.getJsonFacetingResponse(); @@ -204,7 +199,7 @@ public void testMultiRangeFacet() throws Exception { "}"); final DirectJsonQueryRequest request = new DirectJsonQueryRequest(jsonBody); - QueryResponse response = request.process(solrTestRule.getSolrClient(), COLLECTION_NAME); + QueryResponse response = request.process(solrTestRule.getSolrClient()); assertExpectedDocumentsFoundAndReturned(response, NUM_TECHPRODUCTS_DOCS, 10); final NestableJsonFacet topLevelFacetData = response.getJsonFacetingResponse(); @@ -239,7 +234,7 @@ public void testSingleStatFacet() throws Exception { "}"); final DirectJsonQueryRequest request = new DirectJsonQueryRequest(jsonBody); - QueryResponse response = request.process(solrTestRule.getSolrClient(), COLLECTION_NAME); + QueryResponse response = request.process(solrTestRule.getSolrClient()); assertExpectedDocumentsFoundAndReturned(response, NUM_TECHPRODUCTS_DOCS, 10); final NestableJsonFacet topLevelFacetData = response.getJsonFacetingResponse(); @@ -260,7 +255,7 @@ public void testMultiStatFacet() throws Exception { "}"); final DirectJsonQueryRequest request = new DirectJsonQueryRequest(jsonBody); - QueryResponse response = request.process(solrTestRule.getSolrClient(), COLLECTION_NAME); + QueryResponse response = request.process(solrTestRule.getSolrClient()); assertExpectedDocumentsFoundAndReturned(response, NUM_TECHPRODUCTS_DOCS, 10); final NestableJsonFacet topLevelFacetData = response.getJsonFacetingResponse(); @@ -286,7 +281,7 @@ public void testMultiFacetsMixedTypes() throws Exception { "}"); final DirectJsonQueryRequest request = new DirectJsonQueryRequest(jsonBody); - QueryResponse response = request.process(solrTestRule.getSolrClient(), COLLECTION_NAME); + QueryResponse response = request.process(solrTestRule.getSolrClient()); assertExpectedDocumentsFoundAndReturned(response, NUM_TECHPRODUCTS_DOCS, 10); final NestableJsonFacet topLevelFacetData = response.getJsonFacetingResponse(); @@ -324,7 +319,7 @@ public void testNestedTermsFacet() throws Exception { "}"); final DirectJsonQueryRequest request = new DirectJsonQueryRequest(jsonBody); - QueryResponse response = request.process(solrTestRule.getSolrClient(), COLLECTION_NAME); + QueryResponse response = request.process(solrTestRule.getSolrClient()); assertExpectedDocumentsFoundAndReturned(response, NUM_TECHPRODUCTS_DOCS, 10); final NestableJsonFacet topLevelFacetData = response.getJsonFacetingResponse(); @@ -364,7 +359,7 @@ public void testNestedFacetsOfMixedTypes() throws Exception { "}"); final DirectJsonQueryRequest request = new DirectJsonQueryRequest(jsonBody); - QueryResponse response = request.process(solrTestRule.getSolrClient(), COLLECTION_NAME); + QueryResponse response = request.process(solrTestRule.getSolrClient()); assertExpectedDocumentsFoundAndReturned(response, NUM_TECHPRODUCTS_DOCS, 10); final NestableJsonFacet topLevelFacetData = response.getJsonFacetingResponse(); @@ -403,7 +398,7 @@ public void testFacetWithDomainFilteredBySimpleQueryString() throws Exception { "}"); final DirectJsonQueryRequest request = new DirectJsonQueryRequest(jsonBody); - QueryResponse response = request.process(solrTestRule.getSolrClient(), COLLECTION_NAME); + QueryResponse response = request.process(solrTestRule.getSolrClient()); assertExpectedDocumentsFoundAndReturned(response, NUM_TECHPRODUCTS_DOCS, 10); final NestableJsonFacet topLevelFacetData = response.getJsonFacetingResponse(); @@ -434,7 +429,7 @@ public void testFacetWithDomainFilteredByLocalParamsQueryString() throws Excepti " }", "}"); final DirectJsonQueryRequest request = new DirectJsonQueryRequest(jsonBody); - QueryResponse response = request.process(solrTestRule.getSolrClient(), COLLECTION_NAME); + QueryResponse response = request.process(solrTestRule.getSolrClient()); assertExpectedDocumentsFoundAndReturned(response, NUM_TECHPRODUCTS_DOCS, 10); final NestableJsonFacet topLevelFacetData = response.getJsonFacetingResponse(); @@ -466,7 +461,7 @@ public void testFacetWithArbitraryDomainFromQueryString() throws Exception { "}"); final DirectJsonQueryRequest request = new DirectJsonQueryRequest(jsonBody); - QueryResponse response = request.process(solrTestRule.getSolrClient(), COLLECTION_NAME); + QueryResponse response = request.process(solrTestRule.getSolrClient()); assertExpectedDocumentsFoundAndReturned(response, NUM_ELECTRONICS, 10); final NestableJsonFacet topLevelFacetData = response.getJsonFacetingResponse(); @@ -497,7 +492,7 @@ public void testFacetWithArbitraryDomainFromLocalParamsQuery() throws Exception "}"); final DirectJsonQueryRequest request = new DirectJsonQueryRequest(jsonBody); - QueryResponse response = request.process(solrTestRule.getSolrClient(), COLLECTION_NAME); + QueryResponse response = request.process(solrTestRule.getSolrClient()); assertExpectedDocumentsFoundAndReturned(response, NUM_ELECTRONICS, 10); final NestableJsonFacet topLevelFacetData = response.getJsonFacetingResponse(); @@ -527,7 +522,7 @@ public void testFacetWithMultipleSimpleQueryClausesInArbitraryDomain() throws Ex "}"); final DirectJsonQueryRequest request = new DirectJsonQueryRequest(jsonBody); - QueryResponse response = request.process(solrTestRule.getSolrClient(), COLLECTION_NAME); + QueryResponse response = request.process(solrTestRule.getSolrClient()); assertExpectedDocumentsFoundAndReturned(response, NUM_ELECTRONICS, 10); final NestableJsonFacet topLevelFacetData = response.getJsonFacetingResponse(); @@ -557,7 +552,7 @@ public void testFacetWithMultipleLocalParamsQueryClausesInArbitraryDomain() thro "}"); final DirectJsonQueryRequest request = new DirectJsonQueryRequest(jsonBody); - QueryResponse response = request.process(solrTestRule.getSolrClient(), COLLECTION_NAME); + QueryResponse response = request.process(solrTestRule.getSolrClient()); assertExpectedDocumentsFoundAndReturned(response, NUM_ELECTRONICS, 10); final NestableJsonFacet topLevelFacetData = response.getJsonFacetingResponse(); @@ -594,7 +589,7 @@ public void testFacetWithDomainWidenedUsingExcludeTagsToIgnoreFilters() throws E "}"); final DirectJsonQueryRequest request = new DirectJsonQueryRequest(jsonBody); - QueryResponse response = request.process(solrTestRule.getSolrClient(), COLLECTION_NAME); + QueryResponse response = request.process(solrTestRule.getSolrClient()); assertExpectedDocumentsFoundAndReturned(response, NUM_IN_STOCK, 10); final NestableJsonFacet topLevelFacetData = response.getJsonFacetingResponse(); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/response/InputStreamResponseParserTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/response/InputStreamResponseParserTest.java index 3332e82aa1ce..984d19864d49 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/response/InputStreamResponseParserTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/response/InputStreamResponseParserTest.java @@ -53,11 +53,8 @@ private static InputStream getResponse() { public static void beforeTest() throws Exception { EnvUtils.setProperty( ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME.toAbsolutePath().toString()); - solrTestRule.startSolr(createTempDir()); - solrTestRule - .newCollection(DEFAULT_TEST_COLLECTION_NAME) - .withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET) - .create(); + solrTestRule.startSolr(); + solrTestRule.newCollection().withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET).create(); } @Before diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/response/TestSuggesterResponse.java b/solr/solrj/src/test/org/apache/solr/client/solrj/response/TestSuggesterResponse.java index cf66ac7a63f7..d159bcc53284 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/response/TestSuggesterResponse.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/response/TestSuggesterResponse.java @@ -45,11 +45,8 @@ public class TestSuggesterResponse extends SolrTestCaseJ4 { public static void beforeClass() throws Exception { EnvUtils.setProperty( ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME.toAbsolutePath().toString()); - solrTestRule.startSolr(createTempDir()); - solrTestRule - .newCollection(DEFAULT_TEST_COLLECTION_NAME) - .withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET) - .create(); + solrTestRule.startSolr(); + solrTestRule.newCollection().withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET).create(); } static String field = "cat"; diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java index 3a4cbe2532a2..3ff7e74aaa29 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java @@ -2114,6 +2114,7 @@ public Map> invertField(Map model, * using {@code this.getClass()}. */ public static Path getFile(String name) { + // see if it's a classpath resource final URL url = SolrTestCaseJ4.class .getClassLoader() @@ -2128,10 +2129,13 @@ public static Path getFile(String name) { + name); } } + + // see if it's a file path resource final Path file = Path.of(name); if (Files.exists(file)) { - return file; + return file.toAbsolutePath(); // absolute to reduce ambiguity } + throw new RuntimeException( "Cannot find resource in classpath or in file-system (relative to CWD): " + file.toAbsolutePath()); diff --git a/solr/test-framework/src/java/org/apache/solr/embedded/JettySolrRunner.java b/solr/test-framework/src/java/org/apache/solr/embedded/JettySolrRunner.java index 7bf21d549641..05405b37d3c8 100644 --- a/solr/test-framework/src/java/org/apache/solr/embedded/JettySolrRunner.java +++ b/solr/test-framework/src/java/org/apache/solr/embedded/JettySolrRunner.java @@ -49,6 +49,7 @@ import java.util.Random; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import org.apache.solr.client.solrj.SolrClient; @@ -683,7 +684,11 @@ public synchronized void stop() throws Exception { QueuedThreadPool qtp = (QueuedThreadPool) server.getThreadPool(); ReservedThreadExecutor rte = qtp.getBean(ReservedThreadExecutor.class); - server.stop(); + try { + server.stop(); + } catch (TimeoutException e) { + log.warn("Jetty server graceful stop timed out; proceeding with forceful cleanup", e); + } // stop timeout is 0, so we will interrupt right away while (!qtp.isStopped()) { diff --git a/solr/test-framework/src/test/org/apache/solr/client/solrj/apache/BasicHttpSolrClientTest.java b/solr/test-framework/src/test/org/apache/solr/client/solrj/apache/BasicHttpSolrClientTest.java index 6e5320ae7ae5..c7367bab551c 100644 --- a/solr/test-framework/src/test/org/apache/solr/client/solrj/apache/BasicHttpSolrClientTest.java +++ b/solr/test-framework/src/test/org/apache/solr/client/solrj/apache/BasicHttpSolrClientTest.java @@ -103,10 +103,7 @@ public static void beforeTest() throws Exception { EnvUtils.setProperty( ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME.toAbsolutePath().toString()); solrTestRule.startSolr(createTempDir(), new Properties(), jettyConfig); - solrTestRule - .newCollection(DEFAULT_TEST_COLLECTION_NAME) - .withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET) - .create(); + solrTestRule.newCollection().withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET).create(); } @Test diff --git a/solr/test-framework/src/test/org/apache/solr/client/solrj/apache/ConcurrentUpdateSolrClientBadInputTest.java b/solr/test-framework/src/test/org/apache/solr/client/solrj/apache/ConcurrentUpdateSolrClientBadInputTest.java index 8fec38bf3188..9522bb62e488 100644 --- a/solr/test-framework/src/test/org/apache/solr/client/solrj/apache/ConcurrentUpdateSolrClientBadInputTest.java +++ b/solr/test-framework/src/test/org/apache/solr/client/solrj/apache/ConcurrentUpdateSolrClientBadInputTest.java @@ -44,11 +44,8 @@ public class ConcurrentUpdateSolrClientBadInputTest extends SolrTestCaseJ4 { public static void beforeTest() throws Exception { EnvUtils.setProperty( ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME.toAbsolutePath().toString()); - solrTestRule.startSolr(createTempDir()); - solrTestRule - .newCollection(DEFAULT_TEST_COLLECTION_NAME) - .withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET) - .create(); + solrTestRule.startSolr(); + solrTestRule.newCollection().withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET).create(); } @Test diff --git a/solr/test-framework/src/test/org/apache/solr/client/solrj/apache/ConcurrentUpdateSolrClientTest.java b/solr/test-framework/src/test/org/apache/solr/client/solrj/apache/ConcurrentUpdateSolrClientTest.java index 48dc85a8d034..804580f6c2b6 100644 --- a/solr/test-framework/src/test/org/apache/solr/client/solrj/apache/ConcurrentUpdateSolrClientTest.java +++ b/solr/test-framework/src/test/org/apache/solr/client/solrj/apache/ConcurrentUpdateSolrClientTest.java @@ -143,10 +143,7 @@ public static void beforeTest() throws Exception { EnvUtils.setProperty( ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME.toAbsolutePath().toString()); solrTestRule.startSolr(createTempDir(), new Properties(), jettyConfig); - solrTestRule - .newCollection(DEFAULT_TEST_COLLECTION_NAME) - .withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET) - .create(); + solrTestRule.newCollection().withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET).create(); } @Test diff --git a/solr/test-framework/src/test/org/apache/solr/client/solrj/apache/HttpSolrClientConPoolTest.java b/solr/test-framework/src/test/org/apache/solr/client/solrj/apache/HttpSolrClientConPoolTest.java index cbce7fe26cfb..1bd221afad72 100644 --- a/solr/test-framework/src/test/org/apache/solr/client/solrj/apache/HttpSolrClientConPoolTest.java +++ b/solr/test-framework/src/test/org/apache/solr/client/solrj/apache/HttpSolrClientConPoolTest.java @@ -53,19 +53,13 @@ public class HttpSolrClientConPoolTest extends SolrTestCaseJ4 { public static void beforeTest() throws SolrServerException, IOException { EnvUtils.setProperty( ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME.toAbsolutePath().toString()); - solrTestRule.startSolr(createTempDir()); - solrTestRule - .newCollection(DEFAULT_TEST_COLLECTION_NAME) - .withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET) - .create(); + solrTestRule.startSolr(); + solrTestRule.newCollection().withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET).create(); fooUrl = solrTestRule.getBaseUrl(); - secondJetty.startSolr(createTempDir()); - secondJetty - .newCollection(DEFAULT_TEST_COLLECTION_NAME) - .withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET) - .create(); + secondJetty.startSolr(); + secondJetty.newCollection().withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET).create(); barUrl = secondJetty.getBaseUrl(); }