Merge branch 'main' into downsampling++/add-basic-telemetry

Author: gmarouli

docs/changelog/134231.yaml

@@ -0,0 +1,6 @@
+pr: 134231
+summary: Fix unnecessary determinization in index pattern conflict checks
+area: Indices APIs
+type: bug
+issues:
+ - 133652

docs/reference/query-languages/esql/images/functions/knn.svg

@@ -483,9 +483,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.inference.rerank.RerankOperatorTests
   method: testSimpleCircuitBreaking
   issue: https://github.com/elastic/elasticsearch/issues/133619
-- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeIT
-  method: test
-  issue: https://github.com/elastic/elasticsearch/issues/133077
 - class: org.elasticsearch.xpack.kql.parser.KqlParserBooleanQueryTests
   method: testParseOrQuery
   issue: https://github.com/elastic/elasticsearch/issues/133863

muted-tests.yml

@@ -1077,12 +1077,16 @@ static Map<String, List<String>> findConflictingV2Templates(
         boolean checkPriority,
         long priority
     ) {
-        Automaton v1automaton = Regex.simpleMatchToAutomaton(indexPatterns.toArray(Strings.EMPTY_ARRAY));
+        // No need to determinize the automaton, as it is only used to check for intersection with another automaton.
+        // Determinization is avoided because it can fail or become very costly due to state explosion.
+        Automaton v1automaton = Regex.simpleMatchToNonDeterminizedAutomaton(indexPatterns.toArray(Strings.EMPTY_ARRAY));
         Map<String, List<String>> overlappingTemplates = new TreeMap<>();
         for (Map.Entry<String, ComposableIndexTemplate> entry : templatesV2.entrySet()) {
             String name = entry.getKey();
             ComposableIndexTemplate template = entry.getValue();
-            Automaton v2automaton = Regex.simpleMatchToAutomaton(template.indexPatterns().toArray(Strings.EMPTY_ARRAY));
+            // No need to determinize the automaton, as it is only used to check for intersection with another automaton.
+            // Determinization is avoided because it can fail or become very costly due to state explosion.
+            Automaton v2automaton = Regex.simpleMatchToNonDeterminizedAutomaton(template.indexPatterns().toArray(Strings.EMPTY_ARRAY));
             if (Operations.isEmpty(Operations.intersection(v1automaton, v2automaton)) == false) {
                 if (checkPriority == false || priority == template.priorityOrZero()) {
                     logger.debug(

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -2526,6 +2526,23 @@ public void testV2TemplateOverlaps() throws Exception {
         }
     }
 
+    /**
+     * test that using complex index patterns doesn't run into a too_complex_to_determinize_exception,
+     * see https://github.com/elastic/elasticsearch/issues/133652
+     */
+    public void testFindConflictingTemplates_complex_pattern() throws Exception {
+        ProjectMetadata initialProject = ProjectMetadata.builder(randomProjectIdOrDefault()).build();
+        List<String> complexPattern = new ArrayList<>();
+        for (int i = 1; i < 20; i++) {
+            complexPattern.add("cluster-somenamespace-*-app" + i + "*");
+        }
+        ComposableIndexTemplate template = ComposableIndexTemplate.builder().indexPatterns(complexPattern).build();
+        MetadataIndexTemplateService service = getMetadataIndexTemplateService();
+        ProjectMetadata project = service.addIndexTemplateV2(initialProject, false, "foo", template);
+        assertEquals(0, MetadataIndexTemplateService.findConflictingV1Templates(project, "foo", complexPattern).size());
+        assertEquals(0, MetadataIndexTemplateService.findConflictingV2Templates(project, "foo", complexPattern).size());
+    }
+
     /**
      * Tests to add two component templates but ignores both with is valid
      *

server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java

@@ -25,6 +25,7 @@
 import io.opentelemetry.sdk.resources.Resource;
 
 import org.elasticsearch.client.Request;
+import org.elasticsearch.common.hash.BufferedMurmur3Hasher;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -236,6 +237,81 @@ public void testCounterMonotonicity() throws Exception {
         assertThat(ObjectPath.evaluate(metrics, "up_down_counter_delta.time_series_metric"), equalTo("gauge"));
     }
 
+    public void testTsidForBulkIsSame() throws Exception {
+        // This test is to ensure that the _tsid is the same when indexing via a bulk request or OTLP.
+        long now = Clock.getDefault().now();
+
+        export(
+            List.of(
+                createDoubleGauge(
+                    TEST_RESOURCE,
+                    Attributes.builder()
+                        .put("string", "foo")
+                        .put("string_array", "foo", "bar", "baz")
+                        .put("boolean", true)
+                        .put("long", 42L)
+                        .put("double", 42.0)
+                        .put("host.ip", "127.0.0.1")
+                        .build(),
+                    "metric",
+                    42,
+                    "By",
+                    now
+                )
+            )
+        );
+        BufferedMurmur3Hasher hasher = new BufferedMurmur3Hasher(0);
+        hasher.addString("metric");
+        String metricNamesHash = Long.toHexString(hasher.digestHash().hashCode());
+        // Index the same metric via a bulk request
+        Request bulkRequest = new Request("POST", "metrics-generic.otel-default/_bulk?refresh");
+        bulkRequest.setJsonEntity(
+            "{\"create\":{}}\n"
+                + """
+                    {
+                        "@timestamp": $time,
+                        "start_timestamp": $time,
+                        "data_stream": {
+                            "type": "metrics",
+                            "dataset": "generic.otel",
+                            "namespace": "default"
+                        },
+                        "_metric_names_hash": "$metric_names_hash",
+                        "metrics": {
+                            "metric": 42.0
+                        },
+                        "attributes": {
+                            "string": "foo",
+                            "string_array": ["foo", "bar", "baz"],
+                            "boolean": true,
+                            "long": 42,
+                            "double": 42.0,
+                            "host.ip": "127.0.0.1"
+                        },
+                        "resource": {
+                            "attributes": {
+                                "service.name": "elasticsearch"
+                            }
+                        },
+                        "scope": {
+                            "name": "io.opentelemetry.example.metrics"
+                        },
+                        "unit": "By"
+                    }
+                    """.replace("\n", "")
+                    .replace("$time", Long.toString(TimeUnit.NANOSECONDS.toMillis(now) + 1))
+                    .replace("$metric_names_hash", metricNamesHash)
+                + "\n"
+        );
+        assertThat(ObjectPath.createFromResponse(client().performRequest(bulkRequest)).evaluate("errors"), equalTo(false));
+
+        ObjectPath searchResponse = ObjectPath.createFromResponse(
+            client().performRequest(new Request("GET", "metrics-generic.otel-default/_search?docvalue_fields=_tsid"))
+        );
+        assertThat(searchResponse.evaluate("hits.total.value"), equalTo(2));
+        assertThat(searchResponse.evaluate("hits.hits.0.fields._tsid"), equalTo(searchResponse.evaluate("hits.hits.1.fields._tsid")));
+    }
+
     private static Map<String, Object> getMapping(String target) throws IOException {
         Map<String, Object> mappings = ObjectPath.createFromResponse(client().performRequest(new Request("GET", target + "/_mapping")))
             .evaluate("");