Merge branch 'main' into multi-project/x-pack/metadata

Author: tvernum

build.gradle

@@ -241,7 +241,7 @@ def generateUpgradeCompatibilityFile = tasks.register("generateUpgradeCompatibil
 }
 
 def upgradeCompatibilityZip = tasks.register("upgradeCompatibilityZip", Zip) {
-  archiveFile.set(project.layout.buildDirectory.file("rolling-upgrade-compatible-${VersionProperties.elasticsearch}.zip"))
+  archiveFile.set(project.layout.buildDirectory.file("distributions/rolling-upgrade-compatible-${VersionProperties.elasticsearch}.zip"))
   from(generateUpgradeCompatibilityFile)
 }
 

docs/changelog/122615.yaml

@@ -0,0 +1,8 @@
+pr: 122615
+summary: Enable synthetic recovery source by default when synthetic source is enabled.
+  Using synthetic recovery source significantly improves indexing performance compared
+  to regular recovery source.
+area: Mapping
+type: enhancement
+issues:
+ - 116726

docs/changelog/123743.yaml

@@ -0,0 +1,5 @@
+pr: 123743
+summary: Adjust exception thrown when unable to load hunspell dict
+area: Analysis
+type: bug
+issues: []

docs/changelog/124001.yaml

@@ -0,0 +1,7 @@
+pr: 124001
+summary: Use a must boolean statement when pushing down to Lucene when scoring is
+  also needed
+area: ES|QL
+type: bug
+issues:
+ - 123967

docs/changelog/124050.yaml

@@ -0,0 +1,5 @@
+pr: 124050
+summary: Use `FallbackSyntheticSourceBlockLoader` for boolean and date fields
+area: Mapping
+type: enhancement
+issues: []

docs/reference/elasticsearch/configuration-reference/node-settings.md

@@ -46,7 +46,7 @@ The following additional roles are available:
 
 * `voting_only`
 
-$$$coordinating-only-node$$$If you leave `node.roles` unset, then the node is considered to be a [coordinating only node](docs-content://deploy-manage/distributed-architecture/clusters-nodes-shards/node-roles.md#coordinating-only-node-role).
+$$$coordinating-only-node$$$If If you set `node.roles` to an empty array (`node.roles: [ ]`), then the node is considered to be a [coordinating only node](docs-content://deploy-manage/distributed-architecture/clusters-nodes-shards/node-roles.md#coordinating-only-node-role).
 
 ::::{important}
 If you set `node.roles`, ensure you specify every node role your cluster needs. Every cluster requires the following node roles:

docs/reference/ingestion-tools/search-connectors/es-connectors-servicenow.md

@@ -205,9 +205,12 @@ The ServiceNow connector supports roles for access control lists (ACLs) to enabl
 
 For services other than these defaults, the connector iterates over access controls with `read` operations and finds the respective roles for those services.
 
+:::{important}
+The ServiceNow connector applies access control at the service (table) level. This means documents within a given ServiceNow table share the same access control settings. Users with permission to a table can access all documents from that table in Elasticsearch.
+:::
+
 ::::{note}
 The ServiceNow connector does not support scripted and conditional permissions.
-
 ::::
 
 

modules/analysis-common/build.gradle

@@ -6,8 +6,6 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
-import org.elasticsearch.gradle.Version
-
 apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
@@ -36,6 +34,7 @@ artifacts {
 
 tasks.named("yamlRestCompatTestTransform").configure { task ->
   task.replaceValueInMatch("tokens.0.token", "absenț", "romanian")
+  task.skipTest("indices.analyze/15_analyze/Custom analyzer is not buildable", "error response changed with #123743")
 }
 
 tasks.named("yamlRestTest").configure {

modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/indices.analyze/15_analyze.yml

@@ -64,11 +64,11 @@
 "Custom analyzer is not buildable":
   - requires:
       test_runner_features: [ capabilities ]
-      reason: This capability required to run test
       capabilities:
-        - method: GET
-          path: /_analyze
-          capabilities: [ wrong_custom_analyzer_returns_400 ]
+        - method: PUT
+          path: /{index}
+          capabilities: [ hunspell_dict_400 ]
+      reason: "bugfix 'hunspell_dict_400' capability required"
 
   - do:
       catch: bad_request
@@ -80,7 +80,3 @@
           filter:
             type: hunspell
             locale: en_US
-
-  - match: { status: 400 }
-  - match: { error.type: illegal_argument_exception }
-  - match: { error.reason: "Can not build a custom analyzer" }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java

@@ -13,52 +13,45 @@
 import org.elasticsearch.ingest.TestIngestDocument;
 import org.elasticsearch.test.ESTestCase;
 
+import java.util.Collections;
 import java.util.Map;
 
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
+import static java.util.Map.entry;
+import static org.hamcrest.Matchers.anEmptyMap;
+import static org.hamcrest.Matchers.is;
 
 /**
  * Test parsing of an eTLD from a FQDN. The list of eTLDs is maintained here:
  *   https://github.com/publicsuffix/list/blob/master/public_suffix_list.dat
- *
- * Effective TLDs (eTLS) are not the same as DNS TLDs. Uses for eTLDs are listed here.
+ * <p>
+ * Effective TLDs (eTLDs) are not the same as DNS TLDs. Uses for eTLDs are listed here:
  *   https://publicsuffix.org/learn/
  */
 public class RegisteredDomainProcessorTests extends ESTestCase {
-    private Map<String, Object> buildEvent(String domain) {
-        return Map.of("domain", domain);
-    }
 
     public void testBasic() throws Exception {
-        testRegisteredDomainProcessor(buildEvent("www.google.com"), "www.google.com", "google.com", "com", "www");
-        testRegisteredDomainProcessor(buildEvent("google.com"), "google.com", "google.com", "com", null);
-        testRegisteredDomainProcessor(buildEvent(""), null, null, null, null);
-        testRegisteredDomainProcessor(buildEvent("."), null, null, null, null);
-        testRegisteredDomainProcessor(buildEvent("$"), null, null, null, null);
-        testRegisteredDomainProcessor(buildEvent("foo.bar.baz"), null, null, null, null);
-        testRegisteredDomainProcessor(buildEvent("www.books.amazon.co.uk"), "www.books.amazon.co.uk", "amazon.co.uk", "co.uk", "www.books");
+        testRegisteredDomainProcessor("www.google.com", "www.google.com", "google.com", "com", "www");
+        testRegisteredDomainProcessor("google.com", "google.com", "google.com", "com", null);
+        testRegisteredDomainProcessor("", null, null, null, null);
+        testRegisteredDomainProcessor(".", null, null, null, null);
+        testRegisteredDomainProcessor("$", null, null, null, null);
+        testRegisteredDomainProcessor("foo.bar.baz", null, null, null, null);
+        testRegisteredDomainProcessor("www.books.amazon.co.uk", "www.books.amazon.co.uk", "amazon.co.uk", "co.uk", "www.books");
         // Verify "com" is returned as the eTLD, for that FQDN or subdomain
-        testRegisteredDomainProcessor(buildEvent("com"), "com", null, "com", null);
-        testRegisteredDomainProcessor(buildEvent("example.com"), "example.com", "example.com", "com", null);
-        testRegisteredDomainProcessor(buildEvent("googleapis.com"), "googleapis.com", "googleapis.com", "com", null);
+        testRegisteredDomainProcessor("com", "com", null, "com", null);
+        testRegisteredDomainProcessor("example.com", "example.com", "example.com", "com", null);
+        testRegisteredDomainProcessor("googleapis.com", "googleapis.com", "googleapis.com", "com", null);
         testRegisteredDomainProcessor(
-            buildEvent("content-autofill.googleapis.com"),
+            "content-autofill.googleapis.com",
             "content-autofill.googleapis.com",
             "googleapis.com",
             "com",
             "content-autofill"
         );
         // Verify "ssl.fastly.net" is returned as the eTLD, for that FQDN or subdomain
+        testRegisteredDomainProcessor("global.ssl.fastly.net", "global.ssl.fastly.net", "global.ssl.fastly.net", "ssl.fastly.net", null);
         testRegisteredDomainProcessor(
-            buildEvent("global.ssl.fastly.net"),
-            "global.ssl.fastly.net",
-            "global.ssl.fastly.net",
-            "ssl.fastly.net",
-            null
-        );
-        testRegisteredDomainProcessor(
-            buildEvent("1.www.global.ssl.fastly.net"),
+            "1.www.global.ssl.fastly.net",
             "1.www.global.ssl.fastly.net",
             "global.ssl.fastly.net",
             "ssl.fastly.net",
@@ -67,76 +60,81 @@ public void testBasic() throws Exception {
     }
 
     public void testUseRoot() throws Exception {
-        Map<String, Object> source = buildEvent("www.google.co.uk");
-
-        String domainField = "domain";
-        String registeredDomainField = "registered_domain";
-        String topLevelDomainField = "top_level_domain";
-        String subdomainField = "subdomain";
-
         var processor = new RegisteredDomainProcessor(null, null, "domain", "", false);
-
-        IngestDocument input = TestIngestDocument.withDefaultVersion(source);
-        IngestDocument output = processor.execute(input);
-
-        String domain = output.getFieldValue(domainField, String.class);
-        assertThat(domain, equalTo("www.google.co.uk"));
-        String registeredDomain = output.getFieldValue(registeredDomainField, String.class);
-        assertThat(registeredDomain, equalTo("google.co.uk"));
-        String eTLD = output.getFieldValue(topLevelDomainField, String.class);
-        assertThat(eTLD, equalTo("co.uk"));
-        String subdomain = output.getFieldValue(subdomainField, String.class);
-        assertThat(subdomain, equalTo("www"));
+        IngestDocument document = TestIngestDocument.withDefaultVersion(Map.of("domain", "www.google.co.uk"));
+        processor.execute(document);
+        assertThat(
+            document.getSource(),
+            is(
+                Map.ofEntries(
+                    entry("domain", "www.google.co.uk"),
+                    entry("registered_domain", "google.co.uk"),
+                    entry("top_level_domain", "co.uk"),
+                    entry("subdomain", "www")
+                )
+            )
+        );
     }
 
     public void testError() throws Exception {
-        IllegalArgumentException e = expectThrows(
-            IllegalArgumentException.class,
-            () -> testRegisteredDomainProcessor(buildEvent("foo.bar.baz"), null, null, null, null, false)
-        );
-        assertThat(e.getMessage(), containsString("unable to set domain information for document"));
-        e = expectThrows(
-            IllegalArgumentException.class,
-            () -> testRegisteredDomainProcessor(buildEvent("$"), null, null, null, null, false)
-        );
-        assertThat(e.getMessage(), containsString("unable to set domain information for document"));
+        var processor = new RegisteredDomainProcessor(null, null, "domain", "", false);
+
+        {
+            IngestDocument document = TestIngestDocument.withDefaultVersion(Map.of("domain", "foo.bar.baz"));
+            IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(document));
+            assertThat(e.getMessage(), is("unable to set domain information for document"));
+            assertThat(document.getSource(), is(Map.of("domain", "foo.bar.baz")));
+        }
+
+        {
+            IngestDocument document = TestIngestDocument.withDefaultVersion(Map.of("domain", "$"));
+            IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(document));
+            assertThat(e.getMessage(), is("unable to set domain information for document"));
+            assertThat(document.getSource(), is(Map.of("domain", "$")));
+        }
     }
 
-    private void testRegisteredDomainProcessor(
-        Map<String, Object> source,
-        String expectedDomain,
-        String expectedRegisteredDomain,
-        String expectedETLD,
-        String expectedSubdomain
-    ) throws Exception {
-        testRegisteredDomainProcessor(source, expectedDomain, expectedRegisteredDomain, expectedETLD, expectedSubdomain, true);
+    public void testIgnoreMissing() throws Exception {
+        {
+            var processor = new RegisteredDomainProcessor(null, null, "domain", "", false);
+            IngestDocument document = TestIngestDocument.withDefaultVersion(Map.of());
+            IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(document));
+            assertThat(e.getMessage(), is("field [domain] not present as part of path [domain]"));
+            assertThat(document.getSource(), is(anEmptyMap()));
+        }
+
+        {
+            var processor = new RegisteredDomainProcessor(null, null, "domain", "", true);
+            IngestDocument document = TestIngestDocument.withDefaultVersion(Collections.singletonMap("domain", null));
+            processor.execute(document);
+            assertThat(document.getSource(), is(Collections.singletonMap("domain", null)));
+        }
     }
 
     private void testRegisteredDomainProcessor(
-        Map<String, Object> source,
+        String fqdn,
         String expectedDomain,
         String expectedRegisteredDomain,
         String expectedETLD,
-        String expectedSubdomain,
-        boolean ignoreMissing
+        String expectedSubdomain
     ) throws Exception {
         String domainField = "url.domain";
         String registeredDomainField = "url.registered_domain";
         String topLevelDomainField = "url.top_level_domain";
         String subdomainField = "url.subdomain";
 
-        var processor = new RegisteredDomainProcessor(null, null, "domain", "url", ignoreMissing);
+        var processor = new RegisteredDomainProcessor(null, null, "domain", "url", true);
 
-        IngestDocument input = TestIngestDocument.withDefaultVersion(source);
-        IngestDocument output = processor.execute(input);
+        IngestDocument document = TestIngestDocument.withDefaultVersion(Map.of("domain", fqdn));
+        processor.execute(document);
 
-        String domain = output.getFieldValue(domainField, String.class, expectedDomain == null);
-        assertThat(domain, equalTo(expectedDomain));
-        String registeredDomain = output.getFieldValue(registeredDomainField, String.class, expectedRegisteredDomain == null);
-        assertThat(registeredDomain, equalTo(expectedRegisteredDomain));
-        String eTLD = output.getFieldValue(topLevelDomainField, String.class, expectedETLD == null);
-        assertThat(eTLD, equalTo(expectedETLD));
-        String subdomain = output.getFieldValue(subdomainField, String.class, expectedSubdomain == null);
-        assertThat(subdomain, equalTo(expectedSubdomain));
+        String domain = document.getFieldValue(domainField, String.class, expectedDomain == null);
+        assertThat(domain, is(expectedDomain));
+        String registeredDomain = document.getFieldValue(registeredDomainField, String.class, expectedRegisteredDomain == null);
+        assertThat(registeredDomain, is(expectedRegisteredDomain));
+        String eTLD = document.getFieldValue(topLevelDomainField, String.class, expectedETLD == null);
+        assertThat(eTLD, is(expectedETLD));
+        String subdomain = document.getFieldValue(subdomainField, String.class, expectedSubdomain == null);
+        assertThat(subdomain, is(expectedSubdomain));
     }
 }