[backport] Fix ClassCastException with MV_EXPAND on missing field (#110096) (#110264)

Author: luigidellaquila

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce;
 import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer.PropagateEmptyRelation;
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
+import org.elasticsearch.xpack.esql.plan.logical.MvExpand;
 import org.elasticsearch.xpack.esql.plan.logical.TopN;
 import org.elasticsearch.xpack.esql.planner.AbstractPhysicalOperationProviders;
 import org.elasticsearch.xpack.esql.planner.PlannerUtils;
@@ -158,6 +159,12 @@ else if (plan instanceof Project project) {
                     plan = new Eval(project.source(), project.child(), new ArrayList<>(nullLiteral.values()));
                     plan = new Project(project.source(), plan, newProjections);
                 }
+            } else if (plan instanceof MvExpand) {
+                // We cannot replace the target (NamedExpression) with a Literal
+                // https://github.com/elastic/elasticsearch/issues/109974
+                // Unfortunately we cannot remove the MvExpand right away, or we'll lose the output field (layout problems)
+                // TODO but this could be a follow-up optimization
+                return plan;
             }
             // otherwise transform fields in place
             else {

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java

@@ -17,6 +17,8 @@
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
 import org.elasticsearch.xpack.esql.parser.EsqlParser;
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
+import org.elasticsearch.xpack.esql.plan.logical.MvExpand;
+import org.elasticsearch.xpack.esql.plan.logical.local.EsqlProject;
 import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
 import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier;
 import org.elasticsearch.xpack.esql.stats.SearchStats;
@@ -181,6 +183,37 @@ public void testMissingFieldInSort() {
         var source = as(limit.child(), EsRelation.class);
     }
 
+    /**
+     * Expects
+     * EsqlProject[[first_name{f}#6]]
+     * \_Limit[1000[INTEGER]]
+     *   \_MvExpand[last_name{f}#9,last_name{r}#15]
+     *     \_Limit[1000[INTEGER]]
+     *       \_EsRelation[test][_meta_field{f}#11, emp_no{f}#5, first_name{f}#6, ge..]
+     */
+    public void testMissingFieldInMvExpand() {
+        var plan = plan("""
+              from test
+            | mv_expand last_name
+            | keep first_name, last_name
+            """);
+
+        var testStats = statsForMissingField("last_name");
+        var localPlan = localPlan(plan, testStats);
+
+        var project = as(localPlan, EsqlProject.class);
+        var projections = project.projections();
+        assertThat(Expressions.names(projections), contains("first_name", "last_name"));
+
+        var limit = as(project.child(), Limit.class);
+        // MvExpand cannot be optimized (yet) because the target NamedExpression cannot be replaced with a NULL literal
+        // https://github.com/elastic/elasticsearch/issues/109974
+        // See LocalLogicalPlanOptimizer.ReplaceMissingFieldWithNull
+        var mvExpand = as(limit.child(), MvExpand.class);
+        var limit2 = as(mvExpand.child(), Limit.class);
+        as(limit2.child(), EsRelation.class);
+    }
+
     /**
      * Expects
      * EsqlProject[[x{r}#3]]