From 785d6f9df208b4408d795a264343244fdc1818d4 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 21 Oct 2024 12:26:36 -0400 Subject: [PATCH 1/2] ESQL: Skip unsupported grapheme cluster test This skips the test for reversing grapheme clusters if the node doesn't support reversing grapheme clusters. Nodes that are using a jdk before 20 won't support reversing grapheme clusters because they don't have https://bugs.openjdk.org/browse/JDK-8292387 This reworks `EsqlCapabilities` so we can easilly register it only if we're on jdk 20: ``` FN_REVERSE_GRAPHEME_CLUSTERS(Runtime.version().feature() < 20), ``` Closes #114537 Closes #114535 Closes #114536 Closes #114558 Closes #114559 Closes #114560 --- .../functions/description/reverse.asciidoc | 4 ++ .../functions/kibana/definition/reverse.json | 1 + .../esql/functions/kibana/docs/reverse.md | 3 ++ .../src/main/resources/string.csv-spec | 1 + .../xpack/esql/action/EsqlCapabilities.java | 47 ++++++++++--------- .../function/scalar/string/Reverse.java | 6 ++- .../elasticsearch/xpack/esql/CsvTests.java | 2 +- 7 files changed, 39 insertions(+), 25 deletions(-) diff --git a/docs/reference/esql/functions/description/reverse.asciidoc b/docs/reference/esql/functions/description/reverse.asciidoc index fbb3f3f6b4d54..1c2cd15a97d12 100644 --- a/docs/reference/esql/functions/description/reverse.asciidoc +++ b/docs/reference/esql/functions/description/reverse.asciidoc @@ -3,3 +3,7 @@ *Description* Returns a new string representing the input string in reverse order. + +NOTE: If Elasticsearch is running with a JDK version less than 20 then this will not properly reverse Grapheme Clusters. +Elastic Cloud the JDK bundled with Elasticsearch all use newer JDKs. But if you've explicitly shifted to an older jdk +then you'll see things like "๐Ÿ‘๐Ÿฝ๐Ÿ˜Š" be reversed to "๐Ÿฝ๐Ÿ‘๐Ÿ˜Š" instead of the correct "๐Ÿ˜Š๐Ÿ‘๐Ÿฝ". diff --git a/docs/reference/esql/functions/kibana/definition/reverse.json b/docs/reference/esql/functions/kibana/definition/reverse.json index 1b222691530f2..1b4b14b0f829c 100644 --- a/docs/reference/esql/functions/kibana/definition/reverse.json +++ b/docs/reference/esql/functions/kibana/definition/reverse.json @@ -3,6 +3,7 @@ "type" : "eval", "name" : "reverse", "description" : "Returns a new string representing the input string in reverse order.", + "note" : "If Elasticsearch is running with a JDK version less than 20 then this will not properly reverse Grapheme Clusters.\nElastic Cloud the JDK bundled with Elasticsearch all use newer JDKs. But if you've explicitly shifted to an older jdk\nthen you'll see things like \"\uD83D\uDC4D\uD83C\uDFFD\uD83D\uDE0A\" be reversed to \"\uD83C\uDFFD\uD83D\uDC4D\uD83D\uDE0A\" instead of the correct \"\uD83D\uDE0A\uD83D\uDC4D\uD83C\uDFFD\".", "signatures" : [ { "params" : [ diff --git a/docs/reference/esql/functions/kibana/docs/reverse.md b/docs/reference/esql/functions/kibana/docs/reverse.md index cbeade9189d80..7106cf6cfb932 100644 --- a/docs/reference/esql/functions/kibana/docs/reverse.md +++ b/docs/reference/esql/functions/kibana/docs/reverse.md @@ -8,3 +8,6 @@ Returns a new string representing the input string in reverse order. ``` ROW message = "Some Text" | EVAL message_reversed = REVERSE(message); ``` +Note: If Elasticsearch is running with a JDK version less than 20 then this will not properly reverse Grapheme Clusters. +Elastic Cloud the JDK bundled with Elasticsearch all use newer JDKs. But if you've explicitly shifted to an older jdk +then you'll see things like "๐Ÿ‘๐Ÿฝ๐Ÿ˜Š" be reversed to "๐Ÿฝ๐Ÿ‘๐Ÿ˜Š" instead of the correct "๐Ÿ˜Š๐Ÿ‘๐Ÿฝ". diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec index 5313e6630c75d..dd9d519649c01 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec @@ -1236,6 +1236,7 @@ off_on_holiday:keyword | back_home_again:keyword reverseGraphemeClusters required_capability: fn_reverse +required_capability: fn_reverse_grapheme_clusters ROW message = "aฬeฬiฬoฬuฬaฬ€eฬ€iฬ€oฬ€uฬ€aฬ‚eฬ‚iฬ‚oฬ‚uฬ‚๐Ÿ˜Š๐Ÿ‘๐Ÿฝ๐ŸŽ‰๐Ÿ’–เค•เค‚เค เคพเฅ€" | EVAL message_reversed = REVERSE(message); message:keyword | message_reversed:keyword diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index adfba4c487618..62a578ab36776 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -32,6 +32,12 @@ public enum Cap { */ FN_REVERSE, + /** + * Support for reversing whole grapheme clusters. This is not supported + * on JDK versions less than 20. + */ + FN_REVERSE_GRAPHEME_CLUSTERS(Runtime.version().feature() < 20), + /** * Support for function {@code CBRT}. Done in #108574. */ @@ -133,7 +139,7 @@ public enum Cap { * - fixed variable shadowing * - fixed Join.references(), requiring breaking change to Join serialization */ - LOOKUP_V4(true), + LOOKUP_V4(Build.current().isSnapshot()), /** * Support for requesting the "REPEAT" command. @@ -279,7 +285,7 @@ public enum Cap { /** * Support for match operator */ - MATCH_OPERATOR(true), + MATCH_OPERATOR(Build.current().isSnapshot()), /** * Removing support for the {@code META} keyword. @@ -349,7 +355,7 @@ public enum Cap { /** * Supported the text categorization function "CATEGORIZE". */ - CATEGORIZE(true), + CATEGORIZE(Build.current().isSnapshot()), /** * QSTR function @@ -375,7 +381,7 @@ public enum Cap { /** * Support named parameters for field names. */ - NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES(true), + NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES(Build.current().isSnapshot()), /** * Fix sorting not allowed on _source and counters. @@ -397,32 +403,22 @@ public enum Cap { */ FUNCTION_STATS; - private final boolean snapshotOnly; - private final FeatureFlag featureFlag; + private final boolean enabled; Cap() { - this(false, null); + this.enabled = true; }; - Cap(boolean snapshotOnly) { - this(snapshotOnly, null); + Cap(boolean enabled) { + this.enabled = enabled; }; Cap(FeatureFlag featureFlag) { - this(false, featureFlag); - } - - Cap(boolean snapshotOnly, FeatureFlag featureFlag) { - assert featureFlag == null || snapshotOnly == false; - this.snapshotOnly = snapshotOnly; - this.featureFlag = featureFlag; + this.enabled = featureFlag.isEnabled(); } public boolean isEnabled() { - if (featureFlag == null) { - return Build.current().isSnapshot() || this.snapshotOnly == false; - } - return featureFlag.isEnabled(); + return enabled; } public String capabilityName() { @@ -430,12 +426,17 @@ public String capabilityName() { } } - public static final Set CAPABILITIES = capabilities(); + public static final Set CAPABILITIES = capabilities(false); - private static Set capabilities() { + /** + * Get a {@link Set} of all capabilities. If the {@code all} parameter is {@code false} + * then only enabled capabilities are returned - otherwise all + * known capabilities are returned. + */ + public static Set capabilities(boolean all) { List caps = new ArrayList<>(); for (Cap cap : Cap.values()) { - if (cap.isEnabled()) { + if (all || cap.isEnabled()) { caps.add(cap.capabilityName()); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Reverse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Reverse.java index bf4e47d8d0de4..ea111d7ffcc46 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Reverse.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Reverse.java @@ -46,7 +46,11 @@ public class Reverse extends UnaryScalarFunction { file = "string", tag = "reverseEmoji", description = "`REVERSE` works with unicode, too! It keeps unicode grapheme clusters together during reversal." - ) } + ) }, + note = """ + If Elasticsearch is running with a JDK version less than 20 then this will not properly reverse Grapheme Clusters. + Elastic Cloud the JDK bundled with Elasticsearch all use newer JDKs. But if you've explicitly shifted to an older jdk + then you'll see things like "๐Ÿ‘๐Ÿฝ๐Ÿ˜Š" be reversed to "๐Ÿฝ๐Ÿ‘๐Ÿ˜Š" instead of the correct "๐Ÿ˜Š๐Ÿ‘๐Ÿฝ".""" ) public Reverse( Source source, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index ce072e7b0a438..63233f0c46a0d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -257,7 +257,7 @@ public final void test() throws Throwable { assertThat( "Capability is not included in the enabled list capabilities on a snapshot build. Spelling mistake?", testCase.requiredCapabilities, - everyItem(in(EsqlCapabilities.CAPABILITIES)) + everyItem(in(EsqlCapabilities.capabilities(true))) ); } else { for (EsqlCapabilities.Cap c : EsqlCapabilities.Cap.values()) { From 1717f356a9e8e6219f3190120ed9bc0ec82e41ee Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 21 Oct 2024 12:56:16 -0400 Subject: [PATCH 2/2] Update x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Reverse.java Co-authored-by: Bogdan Pintea --- .../xpack/esql/expression/function/scalar/string/Reverse.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Reverse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Reverse.java index ea111d7ffcc46..c20d2aab3a8d6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Reverse.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Reverse.java @@ -49,7 +49,7 @@ public class Reverse extends UnaryScalarFunction { ) }, note = """ If Elasticsearch is running with a JDK version less than 20 then this will not properly reverse Grapheme Clusters. - Elastic Cloud the JDK bundled with Elasticsearch all use newer JDKs. But if you've explicitly shifted to an older jdk + Elastic Cloud and the JDK bundled with Elasticsearch all use newer JDKs. But if you've explicitly shifted to an older jdk then you'll see things like "๐Ÿ‘๐Ÿฝ๐Ÿ˜Š" be reversed to "๐Ÿฝ๐Ÿ‘๐Ÿ˜Š" instead of the correct "๐Ÿ˜Š๐Ÿ‘๐Ÿฝ".""" ) public Reverse(