ESQL: Fix sorts containing _source (#116980) (#117191)

This fixes sorts containing the a `_source` field. It can use the
standard encoder for `BytesRef`s. You can't sort *by* a `_source` field,
but that doesn't really make sense ayway.

Author: nik9000

docs/changelog/116980.yaml

@@ -0,0 +1,6 @@
+pr: 116980
+summary: "ESQL: Fix sorts containing `_source`"
+area: ES|QL
+type: bug
+issues:
+ - 116659

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

@@ -424,6 +424,12 @@ public enum Cap {
          */
         SORTING_ON_SOURCE_AND_COUNTERS_FORBIDDEN,
 
+        /**
+         * Fix {@code SORT} when the {@code _source} field is not a sort key but
+         * <strong>is</strong> being returned.
+         */
+        SORT_RETURNING_SOURCE_OK,
+
         /**
          * Allow filter per individual aggregation.
          */

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java

@@ -362,11 +362,10 @@ private PhysicalOperation planTopN(TopNExec topNExec, LocalExecutionPlannerConte
                 case VERSION -> TopNEncoder.VERSION;
                 case BOOLEAN, NULL, BYTE, SHORT, INTEGER, LONG, DOUBLE, FLOAT, HALF_FLOAT, DATETIME, DATE_NANOS, DATE_PERIOD, TIME_DURATION,
                     OBJECT, SCALED_FLOAT, UNSIGNED_LONG, DOC_DATA_TYPE, TSID_DATA_TYPE -> TopNEncoder.DEFAULT_SORTABLE;
-                case GEO_POINT, CARTESIAN_POINT, GEO_SHAPE, CARTESIAN_SHAPE, COUNTER_LONG, COUNTER_INTEGER, COUNTER_DOUBLE ->
+                case GEO_POINT, CARTESIAN_POINT, GEO_SHAPE, CARTESIAN_SHAPE, COUNTER_LONG, COUNTER_INTEGER, COUNTER_DOUBLE, SOURCE ->
                     TopNEncoder.DEFAULT_UNSORTABLE;
                 // unsupported fields are encoded as BytesRef, we'll use the same encoder; all values should be null at this point
                 case PARTIAL_AGG, UNSUPPORTED -> TopNEncoder.UNSUPPORTED;
-                case SOURCE -> throw new EsqlIllegalArgumentException("No TopN sorting encoder for type " + inverse.get(channel).type());
             };
         }
         List<TopNOperator.SortOrder> orders = topNExec.order().stream().map(order -> {

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/140_metadata.yml

@@ -170,4 +170,43 @@ setup:
       catch: /cannot sort on _source/
       esql.query:
         body:
-          query: 'FROM test metadata _source | sort _source'
+          query: 'FROM test metadata _source | SORT _source'
+
+---
+"sort returning _source is allowed":
+  - requires:
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: []
+          capabilities: [sort_returning_source_ok]
+      reason: "Sorts returning _source should be ok, but weren't in older versions"
+  - do:
+      esql.query:
+        body:
+          query: 'FROM test METADATA _source | SORT case ASC | KEEP case, _source | LIMIT 5'
+  - length: { columns: 2 }
+  - length: { values: 3 }
+  - match: {columns.0.name: "case"}
+  - match: {columns.0.type: "keyword"}
+  - match: {columns.1.name: "_source"}
+  - match: {columns.1.type: "_source"}
+  - match: {values.0.0: "all_ignored"}
+  - match: {values.0.1: {
+    "integer" : "not-an-integer",
+    "keyword" : "long-keyword",
+    "case" : "all_ignored"
+  }}
+  - match: {values.1.0: "integer_ignored"}
+  - match: {values.1.1: {
+    "integer" : "not-an-integer",
+    "keyword" : "ok",
+    "case" : "integer_ignored"
+  }}
+  - match: {values.2.0: "ok"}
+  - match: {values.2.1: {
+    "integer" : 10,
+    "keyword" : "ok",
+    "case" : "ok"
+  }}