From 902fbea482abd39231ba401cc8a8c9f4c4443228 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Tue, 21 Jan 2025 14:22:19 +0100 Subject: [PATCH 1/4] Move scoring in ES|QL out of snapshot (#120354) * Move scoring in ES|QL out of snapshot --------- Co-authored-by: Carlos Delgado <6339205+carlosdelest@users.noreply.github.com> (cherry picked from commit 1b1296ef54e02051898dd351ab8a5fcd3287696b) --- docs/changelog/120354.yaml | 5 + docs/reference/esql/metadata-fields.asciidoc | 2 + .../esql/src/main/antlr/EsqlBaseLexer.tokens | 136 +++++++++--------- .../xpack/esql/action/EsqlCapabilities.java | 2 +- .../xpack/esql/parser/LogicalPlanBuilder.java | 4 +- 5 files changed, 76 insertions(+), 73 deletions(-) create mode 100644 docs/changelog/120354.yaml diff --git a/docs/changelog/120354.yaml b/docs/changelog/120354.yaml new file mode 100644 index 0000000000000..46bdf57327330 --- /dev/null +++ b/docs/changelog/120354.yaml @@ -0,0 +1,5 @@ +pr: 120354 +summary: Move scoring in ES|QL out of snapshot +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/reference/esql/metadata-fields.asciidoc b/docs/reference/esql/metadata-fields.asciidoc index 66046b1b0091f..9617c06cb27c8 100644 --- a/docs/reference/esql/metadata-fields.asciidoc +++ b/docs/reference/esql/metadata-fields.asciidoc @@ -20,6 +20,8 @@ supported ones are: * <>: the ignored source document fields. The field is of the type <>. + * `_score`: when enabled, the final score assigned to each row matching an ES|QL query. Scoring will be updated when using <>. + To enable the access to these fields, the <> source command needs to be provided with a dedicated directive: diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens index 366b455f16402..b1a16987dd8ce 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens @@ -66,68 +66,66 @@ MINUS=65 ASTERISK=66 SLASH=67 PERCENT=68 -LEFT_BRACES=69 -RIGHT_BRACES=70 -NAMED_OR_POSITIONAL_PARAM=71 -OPENING_BRACKET=72 -CLOSING_BRACKET=73 -UNQUOTED_IDENTIFIER=74 -QUOTED_IDENTIFIER=75 -EXPR_LINE_COMMENT=76 -EXPR_MULTILINE_COMMENT=77 -EXPR_WS=78 -EXPLAIN_WS=79 -EXPLAIN_LINE_COMMENT=80 -EXPLAIN_MULTILINE_COMMENT=81 -METADATA=82 -UNQUOTED_SOURCE=83 -FROM_LINE_COMMENT=84 -FROM_MULTILINE_COMMENT=85 -FROM_WS=86 -ID_PATTERN=87 -PROJECT_LINE_COMMENT=88 -PROJECT_MULTILINE_COMMENT=89 -PROJECT_WS=90 -AS=91 -RENAME_LINE_COMMENT=92 -RENAME_MULTILINE_COMMENT=93 -RENAME_WS=94 -ON=95 -WITH=96 -ENRICH_POLICY_NAME=97 -ENRICH_LINE_COMMENT=98 -ENRICH_MULTILINE_COMMENT=99 -ENRICH_WS=100 -ENRICH_FIELD_LINE_COMMENT=101 -ENRICH_FIELD_MULTILINE_COMMENT=102 -ENRICH_FIELD_WS=103 -MVEXPAND_LINE_COMMENT=104 -MVEXPAND_MULTILINE_COMMENT=105 -MVEXPAND_WS=106 -INFO=107 -SHOW_LINE_COMMENT=108 -SHOW_MULTILINE_COMMENT=109 -SHOW_WS=110 -SETTING=111 -SETTING_LINE_COMMENT=112 -SETTTING_MULTILINE_COMMENT=113 -SETTING_WS=114 -LOOKUP_LINE_COMMENT=115 -LOOKUP_MULTILINE_COMMENT=116 -LOOKUP_WS=117 -LOOKUP_FIELD_LINE_COMMENT=118 -LOOKUP_FIELD_MULTILINE_COMMENT=119 -LOOKUP_FIELD_WS=120 -USING=121 -JOIN_LINE_COMMENT=122 -JOIN_MULTILINE_COMMENT=123 -JOIN_WS=124 -METRICS_LINE_COMMENT=125 -METRICS_MULTILINE_COMMENT=126 -METRICS_WS=127 -CLOSING_METRICS_LINE_COMMENT=128 -CLOSING_METRICS_MULTILINE_COMMENT=129 -CLOSING_METRICS_WS=130 +NAMED_OR_POSITIONAL_PARAM=69 +OPENING_BRACKET=70 +CLOSING_BRACKET=71 +UNQUOTED_IDENTIFIER=72 +QUOTED_IDENTIFIER=73 +EXPR_LINE_COMMENT=74 +EXPR_MULTILINE_COMMENT=75 +EXPR_WS=76 +EXPLAIN_WS=77 +EXPLAIN_LINE_COMMENT=78 +EXPLAIN_MULTILINE_COMMENT=79 +METADATA=80 +UNQUOTED_SOURCE=81 +FROM_LINE_COMMENT=82 +FROM_MULTILINE_COMMENT=83 +FROM_WS=84 +ID_PATTERN=85 +PROJECT_LINE_COMMENT=86 +PROJECT_MULTILINE_COMMENT=87 +PROJECT_WS=88 +AS=89 +RENAME_LINE_COMMENT=90 +RENAME_MULTILINE_COMMENT=91 +RENAME_WS=92 +ON=93 +WITH=94 +ENRICH_POLICY_NAME=95 +ENRICH_LINE_COMMENT=96 +ENRICH_MULTILINE_COMMENT=97 +ENRICH_WS=98 +ENRICH_FIELD_LINE_COMMENT=99 +ENRICH_FIELD_MULTILINE_COMMENT=100 +ENRICH_FIELD_WS=101 +MVEXPAND_LINE_COMMENT=102 +MVEXPAND_MULTILINE_COMMENT=103 +MVEXPAND_WS=104 +INFO=105 +SHOW_LINE_COMMENT=106 +SHOW_MULTILINE_COMMENT=107 +SHOW_WS=108 +SETTING=109 +SETTING_LINE_COMMENT=110 +SETTTING_MULTILINE_COMMENT=111 +SETTING_WS=112 +LOOKUP_LINE_COMMENT=113 +LOOKUP_MULTILINE_COMMENT=114 +LOOKUP_WS=115 +LOOKUP_FIELD_LINE_COMMENT=116 +LOOKUP_FIELD_MULTILINE_COMMENT=117 +LOOKUP_FIELD_WS=118 +USING=119 +JOIN_LINE_COMMENT=120 +JOIN_MULTILINE_COMMENT=121 +JOIN_WS=122 +METRICS_LINE_COMMENT=123 +METRICS_MULTILINE_COMMENT=124 +METRICS_WS=125 +CLOSING_METRICS_LINE_COMMENT=126 +CLOSING_METRICS_MULTILINE_COMMENT=127 +CLOSING_METRICS_WS=128 'dissect'=1 'drop'=2 'enrich'=3 @@ -181,10 +179,10 @@ CLOSING_METRICS_WS=130 '*'=66 '/'=67 '%'=68 -']'=73 -'metadata'=82 -'as'=91 -'on'=95 -'with'=96 -'info'=107 -'USING'=121 +']'=71 +'metadata'=80 +'as'=89 +'on'=93 +'with'=94 +'info'=105 +'USING'=119 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index d918d58018afe..402c424962ece 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -599,7 +599,7 @@ public enum Cap { /** * Support the "METADATA _score" directive to enable _score column. */ - METADATA_SCORE(Build.current().isSnapshot()), + METADATA_SCORE, /** * Term function diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java index 3397a5f6adfbc..5719c9dce67ab 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java @@ -18,7 +18,6 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xpack.esql.VerificationException; -import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.common.Failure; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; @@ -280,8 +279,7 @@ public LogicalPlan visitFromCommand(EsqlBaseParser.FromCommandContext ctx) { for (var c : metadataOptionContext.UNQUOTED_SOURCE()) { String id = c.getText(); Source src = source(c); - if (MetadataAttribute.isSupported(id) == false // TODO: drop check below once METADATA_SCORE is no longer snapshot-only - || (EsqlCapabilities.Cap.METADATA_SCORE.isEnabled() == false && MetadataAttribute.SCORE.equals(id))) { + if (MetadataAttribute.isSupported(id) == false) { throw new ParsingException(src, "unsupported metadata field [" + id + "]"); } Attribute a = metadataMap.put(id, MetadataAttribute.create(src, id)); From b2e3b23375089bcd90c2ee05200dd51b5a1a8300 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Tue, 21 Jan 2025 17:47:55 +0100 Subject: [PATCH 2/4] Revert unwanted ES|QL lexer changes from PR #120354 (#120538) * Revert unwanted ES|QL lexer changes from PR #120354 * Update docs/changelog/120538.yaml (cherry picked from commit 04b5b7a8a65c837c93d751a3b1cd65ff6046d1cc) --- docs/changelog/120538.yaml | 5 + .../esql/src/main/antlr/EsqlBaseLexer.tokens | 136 +++++++++--------- 2 files changed, 74 insertions(+), 67 deletions(-) create mode 100644 docs/changelog/120538.yaml diff --git a/docs/changelog/120538.yaml b/docs/changelog/120538.yaml new file mode 100644 index 0000000000000..dfd7e33f4be29 --- /dev/null +++ b/docs/changelog/120538.yaml @@ -0,0 +1,5 @@ +pr: 120538 +summary: "Revert unwanted ES|QL lexer changes from PR #120354" +area: ES|QL +type: bug +issues: [] diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens index b1a16987dd8ce..366b455f16402 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens @@ -66,66 +66,68 @@ MINUS=65 ASTERISK=66 SLASH=67 PERCENT=68 -NAMED_OR_POSITIONAL_PARAM=69 -OPENING_BRACKET=70 -CLOSING_BRACKET=71 -UNQUOTED_IDENTIFIER=72 -QUOTED_IDENTIFIER=73 -EXPR_LINE_COMMENT=74 -EXPR_MULTILINE_COMMENT=75 -EXPR_WS=76 -EXPLAIN_WS=77 -EXPLAIN_LINE_COMMENT=78 -EXPLAIN_MULTILINE_COMMENT=79 -METADATA=80 -UNQUOTED_SOURCE=81 -FROM_LINE_COMMENT=82 -FROM_MULTILINE_COMMENT=83 -FROM_WS=84 -ID_PATTERN=85 -PROJECT_LINE_COMMENT=86 -PROJECT_MULTILINE_COMMENT=87 -PROJECT_WS=88 -AS=89 -RENAME_LINE_COMMENT=90 -RENAME_MULTILINE_COMMENT=91 -RENAME_WS=92 -ON=93 -WITH=94 -ENRICH_POLICY_NAME=95 -ENRICH_LINE_COMMENT=96 -ENRICH_MULTILINE_COMMENT=97 -ENRICH_WS=98 -ENRICH_FIELD_LINE_COMMENT=99 -ENRICH_FIELD_MULTILINE_COMMENT=100 -ENRICH_FIELD_WS=101 -MVEXPAND_LINE_COMMENT=102 -MVEXPAND_MULTILINE_COMMENT=103 -MVEXPAND_WS=104 -INFO=105 -SHOW_LINE_COMMENT=106 -SHOW_MULTILINE_COMMENT=107 -SHOW_WS=108 -SETTING=109 -SETTING_LINE_COMMENT=110 -SETTTING_MULTILINE_COMMENT=111 -SETTING_WS=112 -LOOKUP_LINE_COMMENT=113 -LOOKUP_MULTILINE_COMMENT=114 -LOOKUP_WS=115 -LOOKUP_FIELD_LINE_COMMENT=116 -LOOKUP_FIELD_MULTILINE_COMMENT=117 -LOOKUP_FIELD_WS=118 -USING=119 -JOIN_LINE_COMMENT=120 -JOIN_MULTILINE_COMMENT=121 -JOIN_WS=122 -METRICS_LINE_COMMENT=123 -METRICS_MULTILINE_COMMENT=124 -METRICS_WS=125 -CLOSING_METRICS_LINE_COMMENT=126 -CLOSING_METRICS_MULTILINE_COMMENT=127 -CLOSING_METRICS_WS=128 +LEFT_BRACES=69 +RIGHT_BRACES=70 +NAMED_OR_POSITIONAL_PARAM=71 +OPENING_BRACKET=72 +CLOSING_BRACKET=73 +UNQUOTED_IDENTIFIER=74 +QUOTED_IDENTIFIER=75 +EXPR_LINE_COMMENT=76 +EXPR_MULTILINE_COMMENT=77 +EXPR_WS=78 +EXPLAIN_WS=79 +EXPLAIN_LINE_COMMENT=80 +EXPLAIN_MULTILINE_COMMENT=81 +METADATA=82 +UNQUOTED_SOURCE=83 +FROM_LINE_COMMENT=84 +FROM_MULTILINE_COMMENT=85 +FROM_WS=86 +ID_PATTERN=87 +PROJECT_LINE_COMMENT=88 +PROJECT_MULTILINE_COMMENT=89 +PROJECT_WS=90 +AS=91 +RENAME_LINE_COMMENT=92 +RENAME_MULTILINE_COMMENT=93 +RENAME_WS=94 +ON=95 +WITH=96 +ENRICH_POLICY_NAME=97 +ENRICH_LINE_COMMENT=98 +ENRICH_MULTILINE_COMMENT=99 +ENRICH_WS=100 +ENRICH_FIELD_LINE_COMMENT=101 +ENRICH_FIELD_MULTILINE_COMMENT=102 +ENRICH_FIELD_WS=103 +MVEXPAND_LINE_COMMENT=104 +MVEXPAND_MULTILINE_COMMENT=105 +MVEXPAND_WS=106 +INFO=107 +SHOW_LINE_COMMENT=108 +SHOW_MULTILINE_COMMENT=109 +SHOW_WS=110 +SETTING=111 +SETTING_LINE_COMMENT=112 +SETTTING_MULTILINE_COMMENT=113 +SETTING_WS=114 +LOOKUP_LINE_COMMENT=115 +LOOKUP_MULTILINE_COMMENT=116 +LOOKUP_WS=117 +LOOKUP_FIELD_LINE_COMMENT=118 +LOOKUP_FIELD_MULTILINE_COMMENT=119 +LOOKUP_FIELD_WS=120 +USING=121 +JOIN_LINE_COMMENT=122 +JOIN_MULTILINE_COMMENT=123 +JOIN_WS=124 +METRICS_LINE_COMMENT=125 +METRICS_MULTILINE_COMMENT=126 +METRICS_WS=127 +CLOSING_METRICS_LINE_COMMENT=128 +CLOSING_METRICS_MULTILINE_COMMENT=129 +CLOSING_METRICS_WS=130 'dissect'=1 'drop'=2 'enrich'=3 @@ -179,10 +181,10 @@ CLOSING_METRICS_WS=128 '*'=66 '/'=67 '%'=68 -']'=71 -'metadata'=80 -'as'=89 -'on'=93 -'with'=94 -'info'=105 -'USING'=119 +']'=73 +'metadata'=82 +'as'=91 +'on'=95 +'with'=96 +'info'=107 +'USING'=121 From 5f82c00e8518382d1459b0050e1c8c5b45b20e60 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Mon, 27 Jan 2025 15:24:38 +0100 Subject: [PATCH 3/4] Update docs/changelog/120905.yaml --- docs/changelog/120905.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/120905.yaml diff --git a/docs/changelog/120905.yaml b/docs/changelog/120905.yaml new file mode 100644 index 0000000000000..52db69c596ae6 --- /dev/null +++ b/docs/changelog/120905.yaml @@ -0,0 +1,5 @@ +pr: 120905 +summary: Backport moving scoring in ES|QL out of snapshot into 8.x +area: ES|QL +type: enhancement +issues: [] From 71ff5c609e20adf6cb082ebb6d443448bb89305b Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Mon, 27 Jan 2025 18:15:14 +0100 Subject: [PATCH 4/4] remove unneeded changelog entries --- docs/changelog/120538.yaml | 5 ----- docs/changelog/120905.yaml | 5 ----- 2 files changed, 10 deletions(-) delete mode 100644 docs/changelog/120538.yaml delete mode 100644 docs/changelog/120905.yaml diff --git a/docs/changelog/120538.yaml b/docs/changelog/120538.yaml deleted file mode 100644 index dfd7e33f4be29..0000000000000 --- a/docs/changelog/120538.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 120538 -summary: "Revert unwanted ES|QL lexer changes from PR #120354" -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/120905.yaml b/docs/changelog/120905.yaml deleted file mode 100644 index 52db69c596ae6..0000000000000 --- a/docs/changelog/120905.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 120905 -summary: Backport moving scoring in ES|QL out of snapshot into 8.x -area: ES|QL -type: enhancement -issues: []