Merge branch 'main' into pull/135540

Author: joegallo

docs/changelog/135036.yaml

@@ -0,0 +1,5 @@
+pr: 135036
+summary: Performance improvements for Lookup Join on Expression
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/135039.yaml

@@ -0,0 +1,5 @@
+pr: 135039
+summary: Use optimized field visitor for ignored source queries
+area: Mapping
+type: enhancement
+issues: []

docs/changelog/135393.yaml

@@ -0,0 +1,5 @@
+pr: 135393
+summary: Improve block loader for source only runtime IP fields
+area: Mapping
+type: enhancement
+issues: []

modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexValidator.java

@@ -101,15 +101,6 @@ static CharacterRunAutomaton buildRemoteWhitelist(List<String> whitelist) {
         }
         Automaton automaton = Regex.simpleMatchToAutomaton(whitelist.toArray(Strings.EMPTY_ARRAY));
         automaton = Operations.determinize(automaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
-        if (Operations.isTotal(automaton)) {
-            throw new IllegalArgumentException(
-                "Refusing to start because whitelist "
-                    + whitelist
-                    + " accepts all addresses. "
-                    + "This would allow users to reindex-from-remote any URL they like effectively having Elasticsearch make HTTP GETs "
-                    + "for them."
-            );
-        }
         return new CharacterRunAutomaton(automaton);
     }
 

modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteWhitelistTests.java

@@ -16,7 +16,6 @@
 
 import java.net.UnknownHostException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 
 import static java.util.Collections.emptyList;
@@ -112,35 +111,12 @@ public void testUnwhitelistedRemote() {
         assertEquals("[not in list:" + port + "] not whitelisted in reindex.remote.whitelist", e.getMessage());
     }
 
-    public void testRejectMatchAll() {
-        assertMatchesTooMuch(singletonList("*"));
-        assertMatchesTooMuch(singletonList("**"));
-        assertMatchesTooMuch(singletonList("***"));
-        assertMatchesTooMuch(Arrays.asList("realstuff", "*"));
-        assertMatchesTooMuch(Arrays.asList("*", "realstuff"));
-        List<String> random = randomWhitelist();
-        random.add("*");
-        assertMatchesTooMuch(random);
-    }
-
     public void testIPv6Address() {
         List<String> whitelist = randomWhitelist();
         whitelist.add("[::1]:*");
         checkRemoteWhitelist(buildRemoteWhitelist(whitelist), newRemoteInfo("[::1]", 9200));
     }
 
-    private void assertMatchesTooMuch(List<String> whitelist) {
-        Exception e = expectThrows(IllegalArgumentException.class, () -> buildRemoteWhitelist(whitelist));
-        assertEquals(
-            "Refusing to start because whitelist "
-                + whitelist
-                + " accepts all addresses. "
-                + "This would allow users to reindex-from-remote any URL they like effectively having Elasticsearch make HTTP GETs "
-                + "for them.",
-            e.getMessage()
-        );
-    }
-
     private List<String> randomWhitelist() {
         int size = between(1, 100);
         List<String> whitelist = new ArrayList<>(size);

muted-tests.yml

@@ -615,57 +615,9 @@ tests:
 - class: org.elasticsearch.upgrades.SearchableSnapshotsRollingUpgradeIT
   method: testBlobStoreCacheWithFullCopyInMixedVersions
   issue: https://github.com/elastic/elasticsearch/issues/135474
-- class: org.elasticsearch.upgrades.SyntheticSourceRollingUpgradeIT
-  method: testIndexing {upgradedNodes=3}
-  issue: https://github.com/elastic/elasticsearch/issues/135338
 - class: org.elasticsearch.xpack.esql.qa.multi_node.GenerativeIT
   method: test
   issue: https://github.com/elastic/elasticsearch/issues/134407
-- class: org.elasticsearch.upgrades.LogsUsageRollingUpgradeIT
-  method: testUsage {upgradedNodes=3}
-  issue: https://github.com/elastic/elasticsearch/issues/135312
-- class: org.elasticsearch.upgrades.NoLogsUsageRollingUpgradeIT
-  method: testUsage {upgradedNodes=3}
-  issue: https://github.com/elastic/elasticsearch/issues/135316
-- class: org.elasticsearch.upgrades.LogsdbIndexingRollingUpgradeIT
-  method: testIndexing {upgradedNodes=3}
-  issue: https://github.com/elastic/elasticsearch/issues/135320
-- class: org.elasticsearch.upgrades.TextRollingUpgradeIT
-  method: testIndexing {upgradedNodes=3}
-  issue: https://github.com/elastic/elasticsearch/issues/135237
-- class: org.elasticsearch.upgrades.StandardToLogsDbIndexModeRollingUpgradeIT
-  method: testLogsIndexing {upgradedNodes=3}
-  issue: https://github.com/elastic/elasticsearch/issues/135315
-- class: org.elasticsearch.upgrades.MatchOnlyTextRollingUpgradeIT
-  method: testIndexing {upgradedNodes=3}
-  issue: https://github.com/elastic/elasticsearch/issues/135324
-- class: org.elasticsearch.upgrades.LogsdbIndexingRollingUpgradeIT
-  method: testIndexing {upgradedNodes=2}
-  issue: https://github.com/elastic/elasticsearch/issues/135327
-- class: org.elasticsearch.upgrades.TextRollingUpgradeIT
-  method: testIndexing {upgradedNodes=2}
-  issue: https://github.com/elastic/elasticsearch/issues/135238
-- class: org.elasticsearch.upgrades.TextRollingUpgradeIT
-  method: testIndexing {upgradedNodes=1}
-  issue: https://github.com/elastic/elasticsearch/issues/135236
-- class: org.elasticsearch.upgrades.SyntheticSourceRollingUpgradeIT
-  method: testIndexing {upgradedNodes=2}
-  issue: https://github.com/elastic/elasticsearch/issues/135344
-- class: org.elasticsearch.upgrades.MatchOnlyTextRollingUpgradeIT
-  method: testIndexing {upgradedNodes=2}
-  issue: https://github.com/elastic/elasticsearch/issues/135325
-- class: org.elasticsearch.upgrades.LogsdbIndexingRollingUpgradeIT
-  method: testIndexing {upgradedNodes=1}
-  issue: https://github.com/elastic/elasticsearch/issues/135511
-- class: org.elasticsearch.upgrades.SyntheticSourceRollingUpgradeIT
-  method: testIndexing {upgradedNodes=1}
-  issue: https://github.com/elastic/elasticsearch/issues/135512
-- class: org.elasticsearch.upgrades.MatchOnlyTextRollingUpgradeIT
-  method: testIndexing {upgradedNodes=1}
-  issue: https://github.com/elastic/elasticsearch/issues/135525
-- class: org.elasticsearch.index.codec.vectors.cluster.HierarchicalKMeansTests
-  method: testFewDifferentValues
-  issue: https://github.com/elastic/elasticsearch/issues/135538
 
 # Examples:
 #

qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedFullClusterRestartTestCase.java

@@ -100,9 +100,16 @@ public void extractOldIndexVersion() throws Exception {
         }
     }
 
+    protected void beforeUpgrade() {
+        if (getOldClusterVersion().endsWith("-SNAPSHOT")) {
+            assumeTrue("rename of pattern_text mapper", oldClusterHasFeature("mapper.pattern_text_rename"));
+        }
+    }
+
     @Before
     public void maybeUpgrade() throws Exception {
         if (upgraded == false && requestedUpgradeStatus == UPGRADED) {
+            beforeUpgrade();
             try {
                 if (getOldClusterTestVersion().before(MINIMUM_WIRE_COMPATIBLE_VERSION)) {
                     // First upgrade to latest wire compatible version

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedRollingUpgradeTestCase.java

@@ -54,6 +54,12 @@ public static Iterable<Object[]> parameters() {
 
     protected abstract ElasticsearchCluster getUpgradeCluster();
 
+    protected void beforeUpgrade() {
+        if (getOldClusterVersion().endsWith("-SNAPSHOT")) {
+            assumeTrue("rename of pattern_text mapper", oldClusterHasFeature("mapper.pattern_text_rename"));
+        }
+    }
+
     @Before
     public void upgradeNode() throws Exception {
         // extract old cluster features
@@ -96,6 +102,8 @@ public void upgradeNode() throws Exception {
         // Skip remaining tests if upgrade failed
         assumeFalse("Cluster upgrade failed", upgradeFailed);
 
+        beforeUpgrade();
+
         // finally, upgrade node
         if (upgradedNodes.size() < requestedUpgradedNodes) {
             closeClients();

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -180,11 +180,12 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT = def(9_031_0_00, Version.LUCENE_10_2_2);
     public static final IndexVersion DEFAULT_DENSE_VECTOR_TO_BBQ_HNSW = def(9_032_0_00, Version.LUCENE_10_2_2);
     public static final IndexVersion MATCH_ONLY_TEXT_STORED_AS_BYTES = def(9_033_0_00, Version.LUCENE_10_2_2);
-    public static final IndexVersion IGNORED_SOURCE_FIELDS_PER_ENTRY_WITH_FF = def(9_034_0_00, Version.LUCENE_10_2_2);
+    public static final IndexVersion IGNORED_SOURCE_COALESCED_ENTRIES_WITH_FF = def(9_034_0_00, Version.LUCENE_10_2_2);
     public static final IndexVersion EXCLUDE_SOURCE_VECTORS_DEFAULT = def(9_035_0_00, Version.LUCENE_10_2_2);
     public static final IndexVersion DISABLE_NORMS_BY_DEFAULT_FOR_LOGSDB_AND_TSDB = def(9_036_0_00, Version.LUCENE_10_2_2);
     public static final IndexVersion TSID_CREATED_DURING_ROUTING = def(9_037_0_00, Version.LUCENE_10_2_2);
     public static final IndexVersion UPGRADE_TO_LUCENE_10_3_0 = def(9_038_0_00, Version.LUCENE_10_3_0);
+    public static final IndexVersion IGNORED_SOURCE_COALESCED_ENTRIES = def(9_039_0_00, Version.LUCENE_10_3_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/index/codec/vectors/cluster/HierarchicalKMeans.java

@@ -24,6 +24,7 @@ public class HierarchicalKMeans {
     public static final int MAX_ITERATIONS_DEFAULT = 6;
     public static final int SAMPLES_PER_CLUSTER_DEFAULT = 64;
     public static final float DEFAULT_SOAR_LAMBDA = 1.0f;
+    public static final int NO_SOAR_ASSIGNMENT = -1;
 
     final int dimension;
     final int maxIterations;