Merge branch 'main' into fix-123657

Author: ldematte

docs/changelog/122933.yaml

@@ -0,0 +1,5 @@
+pr: 122933
+summary: Run XPack usage actions on local node
+area: Stats
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java

@@ -289,6 +289,17 @@ void analyzeDocValues(SegmentReader reader, IndexDiskUsageStats stats) throws IO
                     throw new IllegalStateException("Unknown docValues type [" + dvType + "]");
                 }
             }
+            switch (field.docValuesSkipIndexType()) {
+                case NONE -> {
+                }
+                case RANGE -> {
+                    var skipper = docValuesReader.getSkipper(field);
+                    while (skipper.maxDocID(0) != DocIdSetIterator.NO_MORE_DOCS) {
+                        skipper.advance(skipper.maxDocID(skipper.numLevels() - 1) + 1);
+                    }
+                }
+                default -> throw new IllegalStateException("Unknown skipper type [" + field.docValuesSkipIndexType() + "]");
+            }
             stats.addDocValues(field.name, directory.getBytesRead());
         }
     }

server/src/main/java/org/elasticsearch/action/support/local/TransportLocalClusterStateAction.java

@@ -66,6 +66,9 @@ protected abstract void localClusterStateOperation(Task task, Request request, C
 
     @Override
     protected final void doExecute(Task task, Request request, ActionListener<Response> listener) {
+        if (task != null) {
+            request.setParentTask(clusterService.localNode().getId(), task.getId());
+        }
         final var state = clusterService.state();
         final var clusterBlockException = checkBlock(request, state);
         if (clusterBlockException != null) {

server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java

@@ -60,7 +60,7 @@ public class LeaderChecker {
 
     private static final Logger logger = LogManager.getLogger(LeaderChecker.class);
 
-    static final String LEADER_CHECK_ACTION_NAME = "internal:coordination/fault_detection/leader_check";
+    public static final String LEADER_CHECK_ACTION_NAME = "internal:coordination/fault_detection/leader_check";
 
     // the time between checks sent to the leader
     public static final Setting<TimeValue> LEADER_CHECK_INTERVAL_SETTING = Setting.timeSetting(

server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java

@@ -405,6 +405,21 @@ public void testMixedFields() throws Exception {
         }
     }
 
+    public void testDocValuesFieldWithDocValueSkippers() throws Exception {
+        try (Directory dir = createNewDirectory()) {
+            var codecMode = randomFrom(CodecMode.values());
+            indexRandomly(dir, codecMode, between(100, 1000), doc -> addRandomDocValuesField(doc, true));
+            final IndexDiskUsageStats stats = IndexDiskUsageAnalyzer.analyze(testShardId(), lastCommit(dir), () -> {});
+            logger.info("--> stats {}", stats);
+            try (Directory perFieldDir = createNewDirectory()) {
+                rewriteIndexWithPerFieldCodec(dir, codecMode, perFieldDir);
+                final IndexDiskUsageStats perFieldStats = collectPerFieldStats(perFieldDir);
+                assertStats(stats, perFieldStats);
+                assertStats(IndexDiskUsageAnalyzer.analyze(testShardId(), lastCommit(perFieldDir), () -> {}), perFieldStats);
+            }
+        }
+    }
+
     private static void addFieldsToDoc(Document doc, IndexableField[] fields) {
         for (IndexableField field : fields) {
             doc.add(field);
@@ -442,23 +457,27 @@ static void indexRandomly(Directory directory, CodecMode codecMode, int numDocs,
         }
     }
 
-    static void addRandomDocValuesField(Document doc) {
+    static void addRandomDocValuesField(Document doc, boolean indexed) {
         if (randomBoolean()) {
-            doc.add(new NumericDocValuesField("ndv", random().nextInt(1024)));
+            int val = random().nextInt(1024);
+            doc.add(indexed ? NumericDocValuesField.indexedField("ndv", val) : new NumericDocValuesField("ndv", val));
         }
-        if (randomBoolean()) {
+        if (randomBoolean() && indexed == false) {
             doc.add(new BinaryDocValuesField("bdv", new BytesRef(randomAlphaOfLength(3))));
         }
         if (randomBoolean()) {
-            doc.add(new SortedDocValuesField("sdv", new BytesRef(randomAlphaOfLength(3))));
+            var value = new BytesRef(randomAlphaOfLength(3));
+            doc.add(indexed ? SortedDocValuesField.indexedField("sdv", value) : new SortedDocValuesField("sdv", value));
         }
         int numValues = random().nextInt(5);
         for (int i = 0; i < numValues; ++i) {
-            doc.add(new SortedSetDocValuesField("ssdv", new BytesRef(randomAlphaOfLength(3))));
+            var value = new BytesRef(randomAlphaOfLength(3));
+            doc.add(indexed ? SortedSetDocValuesField.indexedField("ssdv", value) : new SortedSetDocValuesField("ssdv", value));
         }
         numValues = random().nextInt(5);
         for (int i = 0; i < numValues; ++i) {
-            doc.add(new SortedNumericDocValuesField("sndv", random().nextInt(1024)));
+            int value = random().nextInt(1024);
+            doc.add(indexed ? SortedNumericDocValuesField.indexedField("sndv", value) : new SortedNumericDocValuesField("sndv", value));
         }
     }
 
@@ -535,7 +554,7 @@ private static float[] randomVector(int dimension) {
 
     static void addRandomFields(Document doc) {
         if (randomBoolean()) {
-            addRandomDocValuesField(doc);
+            addRandomDocValuesField(doc, false);
         }
         if (randomBoolean()) {
             addRandomPostings(doc);

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -134,6 +134,8 @@
 import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.client.RestClient.IGNORE_RESPONSE_CODES_PARAM;
 import static org.elasticsearch.cluster.ClusterState.VERSION_INTRODUCING_TRANSPORT_VERSIONS;
+import static org.elasticsearch.cluster.coordination.FollowersChecker.FOLLOWER_CHECK_ACTION_NAME;
+import static org.elasticsearch.cluster.coordination.LeaderChecker.LEADER_CHECK_ACTION_NAME;
 import static org.elasticsearch.core.Strings.format;
 import static org.elasticsearch.test.MapMatcher.assertMap;
 import static org.elasticsearch.test.MapMatcher.matchesMap;
@@ -258,6 +260,8 @@ public static boolean hasXPack() {
      */
     private static RestClient cleanupClient;
 
+    private static boolean multiProjectEnabled;
+
     public enum ProductFeature {
         XPACK,
         ILM,
@@ -368,6 +372,7 @@ public void initClient() throws IOException {
             availableFeatures = EnumSet.of(ProductFeature.LEGACY_TEMPLATES);
             Set<String> versions = new HashSet<>();
             boolean serverless = false;
+            String multiProjectPluginVariant = null;
 
             for (Map<?, ?> nodeInfo : getNodesInfo(adminClient).values()) {
                 var nodeVersion = nodeInfo.get("version").toString();
@@ -397,6 +402,11 @@ public void initClient() throws IOException {
                     if (moduleName.startsWith("serverless-")) {
                         serverless = true;
                     }
+                    if (moduleName.contains("test-multi-project")) {
+                        multiProjectPluginVariant = "test";
+                    } else if (moduleName.contains("serverless-multi-project")) {
+                        multiProjectPluginVariant = "serverless";
+                    }
                 }
                 if (serverless) {
                     availableFeatures.removeAll(
@@ -418,6 +428,18 @@ public void initClient() throws IOException {
                 .collect(Collectors.toSet());
             assert semanticNodeVersions.isEmpty() == false || serverless;
 
+            if (multiProjectPluginVariant != null) {
+                final Request settingRequest = new Request(
+                    "GET",
+                    "/_cluster/settings?include_defaults&filter_path=*." + multiProjectPluginVariant + ".multi_project.enabled"
+                );
+                settingRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE));
+                final var response = entityAsMap(adminClient.performRequest(settingRequest));
+                multiProjectEnabled = Boolean.parseBoolean(
+                    ObjectPath.evaluate(response, "defaults." + multiProjectPluginVariant + ".multi_project.enabled")
+                );
+            }
+
             testFeatureService = createTestFeatureService(getClusterStateFeatures(adminClient), semanticNodeVersions);
         }
 
@@ -677,6 +699,8 @@ public static void waitForPendingTasks(final RestClient restClient, final Predic
                             final String taskName = line.split("\\s+")[0];
                             if (taskName.startsWith(TransportListTasksAction.TYPE.name())
                                 || taskName.startsWith(HealthNode.TASK_NAME)
+                                || taskName.startsWith(LEADER_CHECK_ACTION_NAME)
+                                || taskName.startsWith(FOLLOWER_CHECK_ACTION_NAME)
                                 || taskFilter.test(taskName)) {
                                 continue;
                             }
@@ -2376,6 +2400,9 @@ public void ensurePeerRecoveryRetentionLeasesRenewedAndSynced(String index) thro
     protected static Map<String, Set<String>> getClusterStateFeatures(RestClient adminClient) throws IOException {
         final Request request = new Request("GET", "_cluster/state");
         request.addParameter("filter_path", "nodes_features");
+        if (multiProjectEnabled) {
+            request.addParameter("multi_project", "true");
+        }
 
         final Response response = adminClient.performRequest(request);
 

x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/AnalyticsUsageTransportAction.java

@@ -38,7 +38,7 @@ public AnalyticsUsageTransportAction(
     }
 
     @Override
-    protected void masterOperation(
+    protected void localClusterStateOperation(
         Task task,
         XPackUsageRequest request,
         ClusterState state,

x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java

@@ -73,7 +73,7 @@ public void testAvailable() throws Exception {
             client
         );
         PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
-        usageAction.masterOperation(task, null, clusterState, future);
+        usageAction.localClusterStateOperation(task, null, clusterState, future);
         XPackFeatureUsage usage = future.get().getUsage();
         assertThat(usage.available(), is(true));
 
@@ -100,7 +100,7 @@ public void testEnabled() throws Exception {
             client
         );
         PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
-        usageAction.masterOperation(task, null, clusterState, future);
+        usageAction.localClusterStateOperation(task, null, clusterState, future);
         XPackFeatureUsage usage = future.get().getUsage();
         assertTrue(usage.enabled());
 

x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CCRUsageTransportAction.java

@@ -48,7 +48,7 @@ public CCRUsageTransportAction(
     }
 
     @Override
-    protected void masterOperation(
+    protected void localClusterStateOperation(
         Task task,
         XPackUsageRequest request,
         ClusterState state,

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java

@@ -152,7 +152,7 @@ public void testUsageStats() throws Exception {
             licenseState
         );
         PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
-        usageAction.masterOperation(null, null, clusterState, future);
+        usageAction.localClusterStateOperation(null, null, clusterState, future);
         CCRInfoTransportAction.Usage ccrUsage = (CCRInfoTransportAction.Usage) future.get().getUsage();
         assertThat(ccrUsage.enabled(), equalTo(true));
         assertThat(ccrUsage.available(), equalTo(false));