Correctly ignore system indices when validationg dot-prefixed indices

Prior to this change, the (incorrect) assumption was that `threadContext.isSystemContext()` returned
true when dealing with system indices. This was eventually revealed to be false (though there is a
separate issue where we aren't emitting deprecation warnings when running locally, but that is a
separate thing).

To fix this, the `DotPrefixValidator` now correctly receives a `SystemIndices` instance and uses the
`findMatchingDescriptor` and `findMatchingDataStreamDescriptor` methods to determine whether a
system index is being referenced. When a matching descriptor is found, the warning emission is
skipped.

Author: dakrone

modules/dot-prefix-validation/build.gradle

@@ -16,7 +16,7 @@ esplugin {
 
 restResources {
   restApi {
-    include '_common', 'indices', 'index', 'cluster', 'nodes', 'get', 'ingest', 'bulk', 'reindex'
+    include '_common', 'indices', 'index', 'cluster', 'nodes', 'get', 'ingest', 'bulk', 'reindex', 'async_search'
   }
 }
 

modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/AutoCreateDotValidator.java

@@ -13,12 +13,13 @@
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.indices.SystemIndices;
 
 import java.util.Set;
 
 public class AutoCreateDotValidator extends DotPrefixValidator<CreateIndexRequest> {
-    public AutoCreateDotValidator(ThreadContext threadContext, ClusterService clusterService) {
-        super(threadContext, clusterService);
+    public AutoCreateDotValidator(ThreadContext threadContext, ClusterService clusterService, SystemIndices systemIndices) {
+        super(threadContext, clusterService, systemIndices);
     }
 
     @Override

modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/CreateIndexDotValidator.java

@@ -13,12 +13,13 @@
 import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.indices.SystemIndices;
 
 import java.util.Set;
 
 public class CreateIndexDotValidator extends DotPrefixValidator<CreateIndexRequest> {
-    public CreateIndexDotValidator(ThreadContext threadContext, ClusterService clusterService) {
-        super(threadContext, clusterService);
+    public CreateIndexDotValidator(ThreadContext threadContext, ClusterService clusterService, SystemIndices systemIndices) {
+        super(threadContext, clusterService, systemIndices);
     }
 
     @Override

modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidationPlugin.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 
@@ -30,12 +31,13 @@ public DotPrefixValidationPlugin() {}
     public Collection<?> createComponents(PluginServices services) {
         ThreadContext context = services.threadPool().getThreadContext();
         ClusterService clusterService = services.clusterService();
+        SystemIndices systemIndices = services.systemIndices();
 
         actionFilters.set(
             List.of(
-                new CreateIndexDotValidator(context, clusterService),
-                new AutoCreateDotValidator(context, clusterService),
-                new IndexTemplateDotValidator(context, clusterService)
+                new CreateIndexDotValidator(context, clusterService, systemIndices),
+                new AutoCreateDotValidator(context, clusterService, systemIndices),
+                new IndexTemplateDotValidator(context, clusterService, systemIndices)
             )
         );
 

modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidator.java

@@ -21,6 +21,7 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.tasks.Task;
 
 import java.util.List;
@@ -94,11 +95,13 @@ public abstract class DotPrefixValidator<RequestType> implements MappedActionFil
     DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(DotPrefixValidator.class);
 
     private final ThreadContext threadContext;
+    private final SystemIndices systemIndices;
     private final boolean isEnabled;
     private volatile Set<Pattern> ignoredIndexPatterns;
 
-    public DotPrefixValidator(ThreadContext threadContext, ClusterService clusterService) {
+    public DotPrefixValidator(ThreadContext threadContext, ClusterService clusterService, SystemIndices systemIndices) {
         this.threadContext = threadContext;
+        this.systemIndices = systemIndices;
         this.isEnabled = VALIDATE_DOT_PREFIXES.get(clusterService.getSettings());
         this.ignoredIndexPatterns = IGNORED_INDEX_PATTERNS_SETTING.get(clusterService.getSettings())
             .stream()
@@ -139,6 +142,10 @@ void validateIndices(@Nullable Set<String> indices) {
                         if (IGNORED_INDEX_NAMES.contains(strippedName)) {
                             return;
                         }
+                        if (systemIndices.findMatchingDescriptor(strippedName) != null
+                            || systemIndices.findMatchingDataStreamDescriptor(strippedName) != null) {
+                            return;
+                        }
                         if (this.ignoredIndexPatterns.stream().anyMatch(p -> p.matcher(strippedName).matches())) {
                             return;
                         }

modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/IndexTemplateDotValidator.java

@@ -12,14 +12,15 @@
 import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.indices.SystemIndices;
 
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Set;
 
 public class IndexTemplateDotValidator extends DotPrefixValidator<TransportPutComposableIndexTemplateAction.Request> {
-    public IndexTemplateDotValidator(ThreadContext threadContext, ClusterService clusterService) {
-        super(threadContext, clusterService);
+    public IndexTemplateDotValidator(ThreadContext threadContext, ClusterService clusterService, SystemIndices systemIndices) {
+        super(threadContext, clusterService, systemIndices);
     }
 
     @Override

modules/dot-prefix-validation/src/test/java/org/elasticsearch/validation/DotPrefixValidatorTests.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.BeforeClass;
@@ -107,7 +108,7 @@ private void assertFails(Set<String> indices) {
     private static class NonOperatorValidator<R> extends DotPrefixValidator<R> {
 
         private NonOperatorValidator() {
-            super(new ThreadContext(Settings.EMPTY), clusterService);
+            super(new ThreadContext(Settings.EMPTY), clusterService, new SystemIndices(List.of()));
         }
 
         @Override

modules/dot-prefix-validation/src/yamlRestTest/resources/rest-api-spec/test/dot_prefix/10_basic.yml

@@ -1,6 +1,11 @@
 ---
 teardown:
+  - requires:
+      test_runner_features: ["allowed_warnings"]
+
   - do:
+      allowed_warnings:
+        - "this request accesses system indices: [.async-search], but in a future major version, direct access to system indices will be prevented by default"
       indices.delete:
         index: .*,-.security-*
 
@@ -194,3 +199,17 @@ teardown:
   - do:
       indices.delete_index_template:
         name: my-template2
+
+---
+"System indices do not cause deprecation warnings":
+  - do:
+      index:
+        index: myindex
+        id: "1"
+        body: {foo: bar}
+
+  - do:
+      async_search.submit:
+        index: myindex
+        keep_alive: 1m
+        keep_on_completion: true