diff --git a/docs/reference/query-languages/esql/_snippets/commands/layout/lookup-join.md b/docs/reference/query-languages/esql/_snippets/commands/layout/lookup-join.md index da99cc69c031a..7b4f2d794ac22 100644 --- a/docs/reference/query-languages/esql/_snippets/commands/layout/lookup-join.md +++ b/docs/reference/query-languages/esql/_snippets/commands/layout/lookup-join.md @@ -42,6 +42,9 @@ results, the output will contain one row for each matching combination. For important information about using `LOOKUP JOIN`, refer to [Usage notes](../../../../esql/esql-lookup-join.md#usage-notes). :::: +:::{include} ../types/lookup-join.md +::: + **Examples** **IP Threat correlation**: This query would allow you to see if any source diff --git a/docs/reference/query-languages/esql/_snippets/commands/types/lookup-join.md b/docs/reference/query-languages/esql/_snippets/commands/types/lookup-join.md new file mode 100644 index 0000000000000..3e54f0ad66277 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/commands/types/lookup-join.md @@ -0,0 +1,21 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| field from the left index | field from the lookup index | +| --- | --- | +| boolean | boolean | +| byte | half_float, float, double, scaled_float, byte, short, integer, long | +| date | date | +| date_nanos | date_nanos | +| double | half_float, float, double, scaled_float, byte, short, integer, long | +| float | half_float, float, double, scaled_float, byte, short, integer, long | +| half_float | half_float, float, double, scaled_float, byte, short, integer, long | +| integer | half_float, float, double, scaled_float, byte, short, integer, long | +| ip | ip | +| keyword | keyword | +| long | half_float, float, double, scaled_float, byte, short, integer, long | +| scaled_float | half_float, float, double, scaled_float, byte, short, integer, long | +| short | half_float, float, double, scaled_float, byte, short, integer, long | +| text | keyword | + diff --git a/docs/reference/query-languages/esql/_snippets/functions/description/knn.md b/docs/reference/query-languages/esql/_snippets/functions/description/knn.md new file mode 100644 index 0000000000000..c39604bbf1fa6 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/description/knn.md @@ -0,0 +1,6 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Description** + +Finds the k nearest vectors to a query vector, as measured by a similarity metric. knn function finds nearest vectors through approximate search on indexed dense_vectors. + diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/knn.md b/docs/reference/query-languages/esql/_snippets/functions/examples/knn.md new file mode 100644 index 0000000000000..2a474d7bfef13 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/knn.md @@ -0,0 +1,30 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Examples** + +```esql +from colors metadata _score +| where knn(rgb_vector, [0, 120, 0]) +| sort _score desc, color asc +``` + +| color:text | rgb_vector:dense_vector | +| --- | --- | +| green | [0.0, 128.0, 0.0] | +| black | [0.0, 0.0, 0.0] | +| olive | [128.0, 128.0, 0.0] | +| teal | [0.0, 128.0, 128.0] | +| lime | [0.0, 255.0, 0.0] | +| sienna | [160.0, 82.0, 45.0] | +| maroon | [128.0, 0.0, 0.0] | +| navy | [0.0, 0.0, 128.0] | +| gray | [128.0, 128.0, 128.0] | +| chartreuse | [127.0, 255.0, 0.0] | + +```esql +from colors metadata _score +| where knn(rgb_vector, [0,255,255], {"k": 4}) +| sort _score desc, color asc +``` + + diff --git a/docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/knn.md b/docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/knn.md new file mode 100644 index 0000000000000..d663e403f8a42 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/knn.md @@ -0,0 +1,19 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported function named parameters** + +`num_candidates` +: (integer) The number of nearest neighbor candidates to consider per shard while doing knn search. Cannot exceed 10,000. Increasing num_candidates tends to improve the accuracy of the final results. Defaults to 1.5 * k + +`boost` +: (float) Floating point number used to decrease or increase the relevance scores of the query.Defaults to 1.0. + +`k` +: (integer) The number of nearest neighbors to return from each shard. Elasticsearch collects k results from each shard, then merges them to find the global top results. This value must be less than or equal to num_candidates. Defaults to 10. + +`rescore_oversample` +: (double) Applies the specified oversampling for rescoring quantized vectors. See [oversampling and rescoring quantized vectors](docs-content://solutions/search/vector/knn.md#dense-vector-knn-search-rescoring) for details. + +`similarity` +: (double) The minimum similarity required for a document to be considered a match. The similarity value calculated relates to the raw similarity used, not the document score. + diff --git a/docs/reference/query-languages/esql/_snippets/functions/layout/knn.md b/docs/reference/query-languages/esql/_snippets/functions/layout/knn.md new file mode 100644 index 0000000000000..dae08f95757b9 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/layout/knn.md @@ -0,0 +1,30 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +## `KNN` [esql-knn] +```{applies_to} +stack: development +serverless: preview +``` + +**Syntax** + +:::{image} ../../../images/functions/knn.svg +:alt: Embedded +:class: text-center +::: + + +:::{include} ../parameters/knn.md +::: + +:::{include} ../description/knn.md +::: + +:::{include} ../types/knn.md +::: + +:::{include} ../functionNamedParams/knn.md +::: + +:::{include} ../examples/knn.md +::: diff --git a/docs/reference/query-languages/esql/_snippets/functions/parameters/knn.md b/docs/reference/query-languages/esql/_snippets/functions/parameters/knn.md new file mode 100644 index 0000000000000..fb1b98a1e8a7a --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/parameters/knn.md @@ -0,0 +1,13 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Parameters** + +`field` +: Field that the query will target. + +`query` +: Vector value to find top nearest neighbours for. + +`options` +: (Optional) kNN additional options as [function named parameters](/reference/query-languages/esql/esql-syntax.md#esql-function-named-params). See [knn query](/reference/query-languages/query-dsl/query-dsl-match-query.md#query-dsl-knn-query) for more information. + diff --git a/docs/reference/query-languages/esql/_snippets/operators/examples/predicates.md b/docs/reference/query-languages/esql/_snippets/operators/examples/predicates.md deleted file mode 100644 index ba34a2d736a83..0000000000000 --- a/docs/reference/query-languages/esql/_snippets/operators/examples/predicates.md +++ /dev/null @@ -1,26 +0,0 @@ -% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. - -**Examples** - -```esql -FROM employees -| WHERE birth_date IS NULL -``` - -| first_name:keyword | last_name:keyword | -| --- | --- | -| Basil | Tramer | -| Florian | Syrotiuk | -| Lucien | Rosenbaum | - -```esql -FROM employees -| WHERE is_rehired IS NOT NULL -| STATS COUNT(emp_no) -``` - -| COUNT(emp_no):long | -| --- | -| 84 | - - diff --git a/docs/reference/query-languages/esql/_snippets/operators/types/predicates.md b/docs/reference/query-languages/esql/_snippets/operators/types/predicates.md deleted file mode 100644 index 19fc0a9465976..0000000000000 --- a/docs/reference/query-languages/esql/_snippets/operators/types/predicates.md +++ /dev/null @@ -1,22 +0,0 @@ -% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. - -**Supported types** - -| field | result | -| --- | --- | -| boolean | boolean | -| cartesian_point | boolean | -| cartesian_shape | boolean | -| date | boolean | -| date_nanos | boolean | -| double | boolean | -| geo_point | boolean | -| geo_shape | boolean | -| integer | boolean | -| ip | boolean | -| keyword | boolean | -| long | boolean | -| text | boolean | -| unsigned_long | boolean | -| version | boolean | - diff --git a/docs/reference/query-languages/esql/esql-lookup-join.md b/docs/reference/query-languages/esql/esql-lookup-join.md index d57437833c1b2..0b6764834e1d6 100644 --- a/docs/reference/query-languages/esql/esql-lookup-join.md +++ b/docs/reference/query-languages/esql/esql-lookup-join.md @@ -142,19 +142,38 @@ Refer to the examples section of the [`LOOKUP JOIN`](/reference/query-languages/ ## Prerequisites [esql-lookup-join-prereqs] -To use `LOOKUP JOIN`, the following requirements must be met: +### Index configuration -* Indices used for lookups must be configured with the [`lookup` index mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting) -* **Compatible data types**: The join key and join field in the lookup index must have compatible data types. This means: - * The data types must either be identical or be internally represented as the same type in {{esql}} - * Numeric types follow these compatibility rules: - * `short` and `byte` are compatible with `integer` (all represented as `int`) - * `float`, `half_float`, and `scaled_float` are compatible with `double` (all represented as `double`) - * For text fields: You can only use text fields as the join key on the left-hand side of the join and only if they have a `.keyword` subfield +Indices used for lookups must be configured with the [`lookup` index mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting). +### Data type compatibility + +Join keys must have compatible data types between the source and lookup indices. Types within the same compatibility group can be joined together: + +| Compatibility group | Types | Notes | +|------------------------|-------------------------------------------------------------------------------------|----------------------------------------------------------------------------------| +| **Numeric family** | `byte`, `short`, `integer`, `long`, `half_float`, `float`, `scaled_float`, `double` | All compatible | +| **Keyword family** | `keyword`, `text.keyword` | Text fields only as join key on left-hand side and must have `.keyword` subfield | +| **Date (Exact)** | `date` | Must match exactly | +| **Date Nanos (Exact)** | `date_nanos` | Must match exactly | +| **Boolean** | `boolean` | Must match exactly | + +```{tip} To obtain a join key with a compatible type, use a [conversion function](/reference/query-languages/esql/functions-operators/type-conversion-functions.md) if needed. +``` -For a complete list of supported data types and their internal representations, see the [Supported Field Types documentation](/reference/query-languages/esql/limitations.md#_supported_types). +### Unsupported Types + +In addition to the [{{esql}} unsupported field types](/reference/query-languages/esql/limitations.md#_unsupported_types), `LOOKUP JOIN` does not support: + +* `VERSION` +* `UNSIGNED_LONG` +* Spatial types like `GEO_POINT`, `GEO_SHAPE` +* Temporal intervals like `DURATION`, `PERIOD` + +```{note} +For a complete list of all types supported in `LOOKUP JOIN`, refer to the [`LOOKUP JOIN` supported types table](/reference/query-languages/esql/commands/processing-commands.md#esql-lookup-join). +``` ## Usage notes diff --git a/docs/reference/query-languages/esql/images/operators/predicates.svg b/docs/reference/query-languages/esql/images/operators/predicates.svg deleted file mode 100644 index 2740ff8dc31e6..0000000000000 --- a/docs/reference/query-languages/esql/images/operators/predicates.svg +++ /dev/null @@ -1 +0,0 @@ -IS NULL and IS NOT NULLv \ No newline at end of file diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/st_geohash.json b/docs/reference/query-languages/esql/kibana/definition/functions/st_geohash.json index d2fc83008c150..43633b336453a 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/st_geohash.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/st_geohash.json @@ -52,4 +52,4 @@ ], "preview" : true, "snapshot_only" : true -} \ No newline at end of file +} diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/st_geohex.json b/docs/reference/query-languages/esql/kibana/definition/functions/st_geohex.json index 9a3a04cb0a7f8..f29db14ed50e7 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/st_geohex.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/st_geohex.json @@ -55,4 +55,4 @@ ], "preview" : true, "snapshot_only" : true -} \ No newline at end of file +} diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/st_geohex_to_long.json b/docs/reference/query-languages/esql/kibana/definition/functions/st_geohex_to_long.json index 52c7918a0c3ad..d582739620024 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/st_geohex_to_long.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/st_geohex_to_long.json @@ -34,4 +34,4 @@ ], "preview" : true, "snapshot_only" : true -} \ No newline at end of file +} diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/st_geohex_to_string.json b/docs/reference/query-languages/esql/kibana/definition/functions/st_geohex_to_string.json index 612b13691d40c..a1abce7c75adb 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/st_geohex_to_string.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/st_geohex_to_string.json @@ -34,4 +34,4 @@ ], "preview" : true, "snapshot_only" : true -} \ No newline at end of file +} diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/st_geotile.json b/docs/reference/query-languages/esql/kibana/definition/functions/st_geotile.json index 06df5e3076fea..d728f186fc5ae 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/st_geotile.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/st_geotile.json @@ -52,4 +52,4 @@ ], "preview" : true, "snapshot_only" : true -} \ No newline at end of file +} diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/st_geotile_to_long.json b/docs/reference/query-languages/esql/kibana/definition/functions/st_geotile_to_long.json index 2eb49b5c320f9..b2c7c01aea606 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/st_geotile_to_long.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/st_geotile_to_long.json @@ -34,4 +34,4 @@ ], "preview" : true, "snapshot_only" : true -} \ No newline at end of file +} diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/st_geotile_to_string.json b/docs/reference/query-languages/esql/kibana/definition/functions/st_geotile_to_string.json index df8e91514dc7b..5a327c2c50976 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/st_geotile_to_string.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/st_geotile_to_string.json @@ -34,4 +34,4 @@ ], "preview" : true, "snapshot_only" : true -} \ No newline at end of file +} diff --git a/docs/reference/query-languages/esql/kibana/definition/operators/is_not_null.json b/docs/reference/query-languages/esql/kibana/definition/operators/is_not_null.json index 3722fbfaddeb8..f184165c249d4 100644 --- a/docs/reference/query-languages/esql/kibana/definition/operators/is_not_null.json +++ b/docs/reference/query-languages/esql/kibana/definition/operators/is_not_null.json @@ -223,6 +223,9 @@ "returnType" : "boolean" } ], + "examples" : [ + "FROM employees\n| WHERE is_rehired IS NOT NULL\n| STATS COUNT(emp_no)" + ], "preview" : false, "snapshot_only" : false } diff --git a/docs/reference/query-languages/esql/kibana/definition/operators/is_null.json b/docs/reference/query-languages/esql/kibana/definition/operators/is_null.json index 0f6e70f8a0b91..dd8c9b4e3d4e7 100644 --- a/docs/reference/query-languages/esql/kibana/definition/operators/is_null.json +++ b/docs/reference/query-languages/esql/kibana/definition/operators/is_null.json @@ -223,6 +223,9 @@ "returnType" : "boolean" } ], + "examples" : [ + "FROM employees\n| WHERE birth_date IS NULL" + ], "preview" : false, "snapshot_only" : false } diff --git a/docs/reference/query-languages/esql/kibana/definition/operators/predicates.json b/docs/reference/query-languages/esql/kibana/definition/operators/predicates.json deleted file mode 100644 index 516343c17af53..0000000000000 --- a/docs/reference/query-languages/esql/kibana/definition/operators/predicates.json +++ /dev/null @@ -1,194 +0,0 @@ -{ - "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.", - "type" : "scalar", - "name" : "predicates", - "description" : "For NULL comparison use the `IS NULL` and `IS NOT NULL` predicates.", - "signatures" : [ - { - "params" : [ - { - "name" : "field", - "type" : "boolean", - "optional" : false, - "description" : "Input value. The input can be a single- or multi-valued column or an expression." - } - ], - "variadic" : false, - "returnType" : "boolean" - }, - { - "params" : [ - { - "name" : "field", - "type" : "cartesian_point", - "optional" : false, - "description" : "Input value. The input can be a single- or multi-valued column or an expression." - } - ], - "variadic" : false, - "returnType" : "boolean" - }, - { - "params" : [ - { - "name" : "field", - "type" : "cartesian_shape", - "optional" : false, - "description" : "Input value. The input can be a single- or multi-valued column or an expression." - } - ], - "variadic" : false, - "returnType" : "boolean" - }, - { - "params" : [ - { - "name" : "field", - "type" : "date", - "optional" : false, - "description" : "Input value. The input can be a single- or multi-valued column or an expression." - } - ], - "variadic" : false, - "returnType" : "boolean" - }, - { - "params" : [ - { - "name" : "field", - "type" : "date_nanos", - "optional" : false, - "description" : "Input value. The input can be a single- or multi-valued column or an expression." - } - ], - "variadic" : false, - "returnType" : "boolean" - }, - { - "params" : [ - { - "name" : "field", - "type" : "double", - "optional" : false, - "description" : "Input value. The input can be a single- or multi-valued column or an expression." - } - ], - "variadic" : false, - "returnType" : "boolean" - }, - { - "params" : [ - { - "name" : "field", - "type" : "geo_point", - "optional" : false, - "description" : "Input value. The input can be a single- or multi-valued column or an expression." - } - ], - "variadic" : false, - "returnType" : "boolean" - }, - { - "params" : [ - { - "name" : "field", - "type" : "geo_shape", - "optional" : false, - "description" : "Input value. The input can be a single- or multi-valued column or an expression." - } - ], - "variadic" : false, - "returnType" : "boolean" - }, - { - "params" : [ - { - "name" : "field", - "type" : "integer", - "optional" : false, - "description" : "Input value. The input can be a single- or multi-valued column or an expression." - } - ], - "variadic" : false, - "returnType" : "boolean" - }, - { - "params" : [ - { - "name" : "field", - "type" : "ip", - "optional" : false, - "description" : "Input value. The input can be a single- or multi-valued column or an expression." - } - ], - "variadic" : false, - "returnType" : "boolean" - }, - { - "params" : [ - { - "name" : "field", - "type" : "keyword", - "optional" : false, - "description" : "Input value. The input can be a single- or multi-valued column or an expression." - } - ], - "variadic" : false, - "returnType" : "boolean" - }, - { - "params" : [ - { - "name" : "field", - "type" : "long", - "optional" : false, - "description" : "Input value. The input can be a single- or multi-valued column or an expression." - } - ], - "variadic" : false, - "returnType" : "boolean" - }, - { - "params" : [ - { - "name" : "field", - "type" : "text", - "optional" : false, - "description" : "Input value. The input can be a single- or multi-valued column or an expression." - } - ], - "variadic" : false, - "returnType" : "boolean" - }, - { - "params" : [ - { - "name" : "field", - "type" : "unsigned_long", - "optional" : false, - "description" : "Input value. The input can be a single- or multi-valued column or an expression." - } - ], - "variadic" : false, - "returnType" : "boolean" - }, - { - "params" : [ - { - "name" : "field", - "type" : "version", - "optional" : false, - "description" : "Input value. The input can be a single- or multi-valued column or an expression." - } - ], - "variadic" : false, - "returnType" : "boolean" - } - ], - "examples" : [ - "FROM employees\n| WHERE birth_date IS NULL", - "FROM employees\n| WHERE is_rehired IS NOT NULL\n| STATS COUNT(emp_no)" - ], - "preview" : false, - "snapshot_only" : false -} diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/bit_length.md b/docs/reference/query-languages/esql/kibana/docs/functions/bit_length.md index 19eb4e805aaf6..6bcd15e35af13 100644 --- a/docs/reference/query-languages/esql/kibana/docs/functions/bit_length.md +++ b/docs/reference/query-languages/esql/kibana/docs/functions/bit_length.md @@ -3,10 +3,11 @@ ### BIT LENGTH Returns the bit length of a string. +Note: All strings are in UTF-8, so a single character can use multiple bytes. + ```esql FROM airports | WHERE country == "India" | KEEP city | EVAL fn_length = LENGTH(city), fn_bit_length = BIT_LENGTH(city) ``` -Note: All strings are in UTF-8, so a single character can use multiple bytes. diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/byte_length.md b/docs/reference/query-languages/esql/kibana/docs/functions/byte_length.md index 6d9a7c9c10c41..a503e9cf54706 100644 --- a/docs/reference/query-languages/esql/kibana/docs/functions/byte_length.md +++ b/docs/reference/query-languages/esql/kibana/docs/functions/byte_length.md @@ -3,10 +3,11 @@ ### BYTE LENGTH Returns the byte length of a string. +Note: All strings are in UTF-8, so a single character can use multiple bytes. + ```esql FROM airports | WHERE country == "India" | KEEP city | EVAL fn_length = LENGTH(city), fn_byte_length = BYTE_LENGTH(city) ``` -Note: All strings are in UTF-8, so a single character can use multiple bytes. diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/ceil.md b/docs/reference/query-languages/esql/kibana/docs/functions/ceil.md index 31830a0b700ac..04d14197753f0 100644 --- a/docs/reference/query-languages/esql/kibana/docs/functions/ceil.md +++ b/docs/reference/query-languages/esql/kibana/docs/functions/ceil.md @@ -3,8 +3,9 @@ ### CEIL Round a number up to the nearest integer. +Note: This is a noop for `long` (including unsigned) and `integer`. For `double` this picks the closest `double` value to the integer similar to [Math.ceil](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/lang/Math.html#ceil(double)). + ```esql ROW a=1.8 | EVAL a=CEIL(a) ``` -Note: This is a noop for `long` (including unsigned) and `integer`. For `double` this picks the closest `double` value to the integer similar to [Math.ceil](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/lang/Math.html#ceil(double)). diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/floor.md b/docs/reference/query-languages/esql/kibana/docs/functions/floor.md index 60cf9cc17ee4c..bdef3d2c31302 100644 --- a/docs/reference/query-languages/esql/kibana/docs/functions/floor.md +++ b/docs/reference/query-languages/esql/kibana/docs/functions/floor.md @@ -3,10 +3,11 @@ ### FLOOR Round a number down to the nearest integer. +Note: This is a noop for `long` (including unsigned) and `integer`. +For `double` this picks the closest `double` value to the integer +similar to [Math.floor](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/lang/Math.html#floor(double)). + ```esql ROW a=1.8 | EVAL a=FLOOR(a) ``` -Note: This is a noop for `long` (including unsigned) and `integer`. -For `double` this picks the closest `double` value to the integer -similar to [Math.floor](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/lang/Math.html#floor(double)). diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/greatest.md b/docs/reference/query-languages/esql/kibana/docs/functions/greatest.md index 752cc0ed77748..5ea02251ed3d1 100644 --- a/docs/reference/query-languages/esql/kibana/docs/functions/greatest.md +++ b/docs/reference/query-languages/esql/kibana/docs/functions/greatest.md @@ -4,8 +4,9 @@ Returns the maximum value from multiple columns. This is similar to [`MV_MAX`](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/mv-functions#esql-mv_max) except it is intended to run on multiple columns at once. +Note: When run on `keyword` or `text` fields, this returns the last string in alphabetical order. When run on `boolean` columns this will return `true` if any values are `true`. + ```esql ROW a = 10, b = 20 | EVAL g = GREATEST(a, b) ``` -Note: When run on `keyword` or `text` fields, this returns the last string in alphabetical order. When run on `boolean` columns this will return `true` if any values are `true`. diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/length.md b/docs/reference/query-languages/esql/kibana/docs/functions/length.md index 71c4dd554f9f7..8e2713e37e073 100644 --- a/docs/reference/query-languages/esql/kibana/docs/functions/length.md +++ b/docs/reference/query-languages/esql/kibana/docs/functions/length.md @@ -3,10 +3,11 @@ ### LENGTH Returns the character length of a string. +Note: All strings are in UTF-8, so a single character can use multiple bytes. + ```esql FROM airports | WHERE country == "India" | KEEP city | EVAL fn_length = LENGTH(city) ``` -Note: All strings are in UTF-8, so a single character can use multiple bytes. diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/median.md b/docs/reference/query-languages/esql/kibana/docs/functions/median.md index 6fc72bfb82421..0028f7afc4934 100644 --- a/docs/reference/query-languages/esql/kibana/docs/functions/median.md +++ b/docs/reference/query-languages/esql/kibana/docs/functions/median.md @@ -3,8 +3,9 @@ ### MEDIAN The value that is greater than half of all values and less than half of all values, also known as the 50% [`PERCENTILE`](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/aggregation-functions#esql-percentile). +Note: Like [`PERCENTILE`](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/aggregation-functions#esql-percentile), `MEDIAN` is [usually approximate](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/aggregation-functions#esql-percentile-approximate). + ```esql FROM employees | STATS MEDIAN(salary), PERCENTILE(salary, 50) ``` -Note: Like [`PERCENTILE`](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/aggregation-functions#esql-percentile), `MEDIAN` is [usually approximate](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/aggregation-functions#esql-percentile-approximate). diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/median_absolute_deviation.md b/docs/reference/query-languages/esql/kibana/docs/functions/median_absolute_deviation.md index 4be62924710f1..94a3c0cb9a3f8 100644 --- a/docs/reference/query-languages/esql/kibana/docs/functions/median_absolute_deviation.md +++ b/docs/reference/query-languages/esql/kibana/docs/functions/median_absolute_deviation.md @@ -5,8 +5,9 @@ Returns the median absolute deviation, a measure of variability. It is a robust It is calculated as the median of each data point’s deviation from the median of the entire sample. That is, for a random variable `X`, the median absolute deviation is `median(|median(X) - X|)`. +Note: Like [`PERCENTILE`](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/aggregation-functions#esql-percentile), `MEDIAN_ABSOLUTE_DEVIATION` is [usually approximate](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/aggregation-functions#esql-percentile-approximate). + ```esql FROM employees | STATS MEDIAN(salary), MEDIAN_ABSOLUTE_DEVIATION(salary) ``` -Note: Like [`PERCENTILE`](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/aggregation-functions#esql-percentile), `MEDIAN_ABSOLUTE_DEVIATION` is [usually approximate](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/aggregation-functions#esql-percentile-approximate). diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/mv_dedupe.md b/docs/reference/query-languages/esql/kibana/docs/functions/mv_dedupe.md index 8c37f24139174..7c330e2c8dd63 100644 --- a/docs/reference/query-languages/esql/kibana/docs/functions/mv_dedupe.md +++ b/docs/reference/query-languages/esql/kibana/docs/functions/mv_dedupe.md @@ -3,8 +3,9 @@ ### MV DEDUPE Remove duplicate values from a multivalued field. +Note: `MV_DEDUPE` may, but won’t always, sort the values in the column. + ```esql ROW a=["foo", "foo", "bar", "foo"] | EVAL dedupe_a = MV_DEDUPE(a) ``` -Note: `MV_DEDUPE` may, but won’t always, sort the values in the column. diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/mv_median_absolute_deviation.md b/docs/reference/query-languages/esql/kibana/docs/functions/mv_median_absolute_deviation.md index 94ed0def6195f..0d4863948c9bf 100644 --- a/docs/reference/query-languages/esql/kibana/docs/functions/mv_median_absolute_deviation.md +++ b/docs/reference/query-languages/esql/kibana/docs/functions/mv_median_absolute_deviation.md @@ -5,8 +5,9 @@ Converts a multivalued field into a single valued field containing the median ab It is calculated as the median of each data point’s deviation from the median of the entire sample. That is, for a random variable `X`, the median absolute deviation is `median(|median(X) - X|)`. +Note: If the field has an even number of values, the medians will be calculated as the average of the middle two values. If the value is not a floating point number, the averages are rounded towards 0. + ```esql ROW values = [0, 2, 5, 6] | EVAL median_absolute_deviation = MV_MEDIAN_ABSOLUTE_DEVIATION(values), median = MV_MEDIAN(values) ``` -Note: If the field has an even number of values, the medians will be calculated as the average of the middle two values. If the value is not a floating point number, the averages are rounded towards 0. diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/pow.md b/docs/reference/query-languages/esql/kibana/docs/functions/pow.md index 8c27e49d397f1..da08d767d7e9b 100644 --- a/docs/reference/query-languages/esql/kibana/docs/functions/pow.md +++ b/docs/reference/query-languages/esql/kibana/docs/functions/pow.md @@ -3,8 +3,9 @@ ### POW Returns the value of `base` raised to the power of `exponent`. +Note: It is still possible to overflow a double result here; in that case, null will be returned. + ```esql ROW base = 2.0, exponent = 2 | EVAL result = POW(base, exponent) ``` -Note: It is still possible to overflow a double result here; in that case, null will be returned. diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/to_date_nanos.md b/docs/reference/query-languages/esql/kibana/docs/functions/to_date_nanos.md index 6d2b18db20680..223ae10b45a4e 100644 --- a/docs/reference/query-languages/esql/kibana/docs/functions/to_date_nanos.md +++ b/docs/reference/query-languages/esql/kibana/docs/functions/to_date_nanos.md @@ -3,10 +3,11 @@ ### TO DATE NANOS Converts an input to a nanosecond-resolution date value (aka date_nanos). +Note: The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z, attempting to convert values outside of that range will result in null with a warning. Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch. + ```esql FROM date_nanos | WHERE MV_MIN(nanos) < TO_DATE_NANOS("2023-10-23T12:27:28.948Z") AND millis > "2000-01-01" | SORT nanos DESC ``` -Note: The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z, attempting to convert values outside of that range will result in null with a warning. Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch. diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/to_datetime.md b/docs/reference/query-languages/esql/kibana/docs/functions/to_datetime.md index b5361583ff465..de94e09965ce2 100644 --- a/docs/reference/query-languages/esql/kibana/docs/functions/to_datetime.md +++ b/docs/reference/query-languages/esql/kibana/docs/functions/to_datetime.md @@ -5,8 +5,9 @@ Converts an input value to a date value. A string will only be successfully converted if it’s respecting the format `yyyy-MM-dd'T'HH:mm:ss.SSS'Z'`. To convert dates in other formats, use [`DATE_PARSE`](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/date-time-functions#esql-date_parse). +Note: Note that when converting from nanosecond resolution to millisecond resolution with this function, the nanosecond date is truncated, not rounded. + ```esql ROW string = ["1953-09-02T00:00:00.000Z", "1964-06-02T00:00:00.000Z", "1964-06-02 00:00:00"] | EVAL datetime = TO_DATETIME(string) ``` -Note: Note that when converting from nanosecond resolution to millisecond resolution with this function, the nanosecond date is truncated, not rounded. diff --git a/docs/reference/query-languages/esql/kibana/docs/operators/add.md b/docs/reference/query-languages/esql/kibana/docs/operators/add.md index b07deb2c38360..0d91dcd400252 100644 --- a/docs/reference/query-languages/esql/kibana/docs/operators/add.md +++ b/docs/reference/query-languages/esql/kibana/docs/operators/add.md @@ -2,4 +2,3 @@ ### ADD `+` Add two numbers together. If either field is [multivalued](https://www.elastic.co/docs/reference/query-languages/esql/esql-multivalued-fields) then the result is `null`. - diff --git a/docs/reference/query-languages/esql/kibana/docs/operators/is_not_null.md b/docs/reference/query-languages/esql/kibana/docs/operators/is_not_null.md index 25157e2bb7af2..d91564c45778d 100644 --- a/docs/reference/query-languages/esql/kibana/docs/operators/is_not_null.md +++ b/docs/reference/query-languages/esql/kibana/docs/operators/is_not_null.md @@ -4,3 +4,9 @@ Returns `false` if the value is `NULL`, `true` otherwise. Note: If a field is only in some documents it will be `NULL` in the documents that did not contain it. + +```esql +FROM employees +| WHERE is_rehired IS NOT NULL +| STATS COUNT(emp_no) +``` diff --git a/docs/reference/query-languages/esql/kibana/docs/operators/is_null.md b/docs/reference/query-languages/esql/kibana/docs/operators/is_null.md index 625b819935332..3d9d05d037516 100644 --- a/docs/reference/query-languages/esql/kibana/docs/operators/is_null.md +++ b/docs/reference/query-languages/esql/kibana/docs/operators/is_null.md @@ -4,3 +4,8 @@ Returns `true` if the value is `NULL`, `false` otherwise. Note: If a field is only in some documents it will be `NULL` in the documents that did not contain it. + +```esql +FROM employees +| WHERE birth_date IS NULL +``` diff --git a/docs/reference/query-languages/esql/kibana/docs/operators/mod.md b/docs/reference/query-languages/esql/kibana/docs/operators/mod.md index 71cad28b1dc23..9a61ddf4b745d 100644 --- a/docs/reference/query-languages/esql/kibana/docs/operators/mod.md +++ b/docs/reference/query-languages/esql/kibana/docs/operators/mod.md @@ -2,4 +2,3 @@ ### MODULO `%` Divide one number by another and return the remainder. If either field is [multivalued](https://www.elastic.co/docs/reference/query-languages/esql/esql-multivalued-fields) then the result is `null`. - diff --git a/docs/reference/query-languages/esql/kibana/docs/operators/mul.md b/docs/reference/query-languages/esql/kibana/docs/operators/mul.md index eaa4cba9d7e7d..97921119d449d 100644 --- a/docs/reference/query-languages/esql/kibana/docs/operators/mul.md +++ b/docs/reference/query-languages/esql/kibana/docs/operators/mul.md @@ -2,4 +2,3 @@ ### MULTIPLY `*` Multiply two numbers together. If either field is [multivalued](https://www.elastic.co/docs/reference/query-languages/esql/esql-multivalued-fields) then the result is `null`. - diff --git a/docs/reference/query-languages/esql/kibana/docs/operators/neg.md b/docs/reference/query-languages/esql/kibana/docs/operators/neg.md index 036c545d247f4..e25b1ede4c47a 100644 --- a/docs/reference/query-languages/esql/kibana/docs/operators/neg.md +++ b/docs/reference/query-languages/esql/kibana/docs/operators/neg.md @@ -2,4 +2,3 @@ ### NEGATE `-` Returns the negation of the argument. - diff --git a/docs/reference/query-languages/esql/kibana/docs/operators/not in.md b/docs/reference/query-languages/esql/kibana/docs/operators/not in.md index 810773cf977ff..d638a876148dd 100644 --- a/docs/reference/query-languages/esql/kibana/docs/operators/not in.md +++ b/docs/reference/query-languages/esql/kibana/docs/operators/not in.md @@ -2,4 +2,3 @@ ### NOT IN The `NOT IN` operator allows testing whether a field or expression does *not* equal any element in a list of literals, fields or expressions. - diff --git a/docs/reference/query-languages/esql/kibana/docs/operators/not like.md b/docs/reference/query-languages/esql/kibana/docs/operators/not like.md index 6164a03110755..dde8a60257346 100644 --- a/docs/reference/query-languages/esql/kibana/docs/operators/not like.md +++ b/docs/reference/query-languages/esql/kibana/docs/operators/not like.md @@ -10,4 +10,3 @@ The following wildcard characters are supported: * `*` matches zero or more characters. * `?` matches one character. - diff --git a/docs/reference/query-languages/esql/kibana/docs/operators/not rlike.md b/docs/reference/query-languages/esql/kibana/docs/operators/not rlike.md index c2b04b4a9de7a..c1b2973f6ffa0 100644 --- a/docs/reference/query-languages/esql/kibana/docs/operators/not rlike.md +++ b/docs/reference/query-languages/esql/kibana/docs/operators/not rlike.md @@ -5,4 +5,3 @@ Use `RLIKE` to filter data based on string patterns using using [regular expressions](https://www.elastic.co/docs/reference/query-languages/query-dsl/regexp-syntax). `RLIKE` usually acts on a field placed on the left-hand side of the operator, but it can also act on a constant (literal) expression. The right-hand side of the operator represents the pattern. - diff --git a/docs/reference/query-languages/esql/kibana/docs/operators/predicates.md b/docs/reference/query-languages/esql/kibana/docs/operators/predicates.md deleted file mode 100644 index f40f821193fe5..0000000000000 --- a/docs/reference/query-languages/esql/kibana/docs/operators/predicates.md +++ /dev/null @@ -1,9 +0,0 @@ -% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. - -### PREDICATES -For NULL comparison use the `IS NULL` and `IS NOT NULL` predicates. - -```esql -FROM employees -| WHERE birth_date IS NULL -``` diff --git a/docs/reference/query-languages/esql/kibana/docs/operators/sub.md b/docs/reference/query-languages/esql/kibana/docs/operators/sub.md index 516cd3841a6a1..1fea90a58ec53 100644 --- a/docs/reference/query-languages/esql/kibana/docs/operators/sub.md +++ b/docs/reference/query-languages/esql/kibana/docs/operators/sub.md @@ -2,4 +2,3 @@ ### SUBTRACT `-` Subtract one number from another. If either field is [multivalued](https://www.elastic.co/docs/reference/query-languages/esql/esql-multivalued-fields) then the result is `null`. - diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle index 0db0e558cd147..447ceb53489f2 100644 --- a/x-pack/plugin/esql/build.gradle +++ b/x-pack/plugin/esql/build.gradle @@ -226,6 +226,47 @@ tasks.named("test").configure { } } +// This is similar to the test task above, but needed for the LookupJoinTypesIT which runs in the internalClusterTest task +// and generates a types table for the LOOKUP JOIN command. It is possible in future we might have move tests that do this. +tasks.named("internalClusterTest").configure { + if (buildParams.ci == false) { + systemProperty 'generateDocs', true + def injected = project.objects.newInstance(Injected) + // Define the folder to delete and recreate + def tempDir = file("build/testrun/internalClusterTest/temp/esql") + doFirst { + injected.fs.delete { + it.delete(tempDir) + } + // Re-create this folder so we can save a table of generated examples to extract from csv-spec tests + tempDir.mkdirs() // Recreate the folder + } + File snippetsFolder = file("build/testrun/internalClusterTest/temp/esql/_snippets") + def snippetsDocFolder = file("${rootDir}/docs/reference/query-languages/esql/_snippets") + def snippetsTree = fileTree(snippetsFolder).matching { + include "**/types/*.md" // Recursively include all types/*.md files (effectively counting functions and operators) + } + + doLast { + def snippets = snippetsTree.files.collect { it.name } + int countSnippets = snippets.size() + if (countSnippets == 0) { + logger.quiet("ESQL Docs: No function/operator snippets created. Skipping sync.") + } else { + logger.quiet("ESQL Docs: Found $countSnippets generated function/operator snippets to patch into docs") + injected.fs.sync { + from snippetsFolder + into snippetsDocFolder + include '**/*.md' + preserve { + include '**/*.md' + } + } + } + } + } +} + /**************************************************************** * Enable QA/rest integration tests for snapshot builds only * * TODO: Enable for all builds upon this feature release * diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java index df021f27a31fe..731b2976f88d8 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java @@ -19,6 +19,8 @@ import org.elasticsearch.xpack.core.esql.action.ColumnInfo; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.DocsV3Support; +import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.plan.logical.join.Join; import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; import org.elasticsearch.xpack.spatial.SpatialPlugin; @@ -36,6 +38,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Consumer; +import java.util.function.Supplier; import java.util.stream.Collectors; import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; @@ -265,6 +268,22 @@ private static boolean existingIndex(Collection existing, DataType return existing.stream().anyMatch(c -> c.exists(indexName)); } + /** This test generates documentation for the supported output types of the lookup join. */ + public void testOutputSupportedTypes() throws Exception { + Map, DataType> signatures = new LinkedHashMap<>(); + for (TestConfigs configs : testConfigurations.values()) { + if (configs.group.equals("unsupported") || configs.group.equals("union-types")) { + continue; + } + for (TestConfig config : configs.configs.values()) { + if (config instanceof TestConfigPasses) { + signatures.put(List.of(config.mainType(), config.lookupType()), null); + } + } + } + saveJoinTypes(() -> signatures); + } + public void testLookupJoinStrings() { testLookupJoinTypes("strings"); } @@ -747,4 +766,18 @@ public void doTest() { private boolean isValidDataType(DataType dataType) { return UNDER_CONSTRUCTION.get(dataType) == null || UNDER_CONSTRUCTION.get(dataType).isEnabled(); } + + private static void saveJoinTypes(Supplier, DataType>> signatures) throws Exception { + ArrayList args = new ArrayList<>(); + args.add(new EsqlFunctionRegistry.ArgSignature("field from the left index", null, null, false, false)); + args.add(new EsqlFunctionRegistry.ArgSignature("field from the lookup index", null, null, false, false)); + DocsV3Support.CommandsDocsSupport docs = new DocsV3Support.CommandsDocsSupport( + "lookup-join", + LookupJoinTypesIT.class, + null, + args, + signatures + ); + docs.renderDocs(); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNotNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNotNull.java index 82818c5a329c0..36486199609a0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNotNull.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNotNull.java @@ -25,6 +25,7 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; +import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; @@ -54,7 +55,8 @@ public class IsNotNull extends UnaryScalarFunction implements EvaluatorMapper, N "unsigned_long", "counter_long", "counter_integer", - "counter_double" } + "counter_double" }, + examples = { @Example(file = "null", tag = "is-not-null") } ) public IsNotNull( Source source, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNull.java index e93124af54186..7106fe2f33272 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNull.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNull.java @@ -26,6 +26,7 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; +import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; @@ -51,7 +52,8 @@ public class IsNull extends UnaryScalarFunction implements EvaluatorMapper, Nega "unsigned_long", "counter_long", "counter_integer", - "counter_double" } + "counter_double" }, + examples = { @Example(file = "null", tag = "is-null") } ) public IsNull( Source source, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java index 7bcb194c62a75..e60eeb63785d5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java @@ -60,7 +60,9 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.TreeMap; import java.util.function.Function; import java.util.function.Supplier; import java.util.regex.Matcher; @@ -308,7 +310,7 @@ public License.OperationMode invoke(List fieldTypes) throws Exception protected final String name; protected final FunctionDefinition definition; protected final Logger logger; - private final Supplier, DataType>> signatures; + protected final Supplier, DataType>> signatures; private TempFileWriter tempFileWriter; private final LicenseRequirementChecker licenseChecker; @@ -836,14 +838,30 @@ void renderDetailedDescription(String detailedDescription, String note) throws I /** Command specific docs generating, currently very empty since we only render kibana definition files */ public static class CommandsDocsSupport extends DocsV3Support { private final LogicalPlan command; + private List args; private final XPackLicenseState licenseState; + /** Used in CommandLicenseTests to generate Kibana docs with licensing information for commands */ public CommandsDocsSupport(String name, Class testClass, LogicalPlan command, XPackLicenseState licenseState) { super("commands", name, testClass, Map::of); this.command = command; this.licenseState = licenseState; } + /** Used in LookupJoinTypesIT to generate table of supported types for join field */ + public CommandsDocsSupport( + String name, + Class testClass, + LogicalPlan command, + List args, + Supplier, DataType>> signatures + ) { + super("commands", name, testClass, signatures); + this.command = command; + this.args = args; + this.licenseState = null; + } + @Override public void renderSignature() throws IOException { // Unimplemented until we make command docs dynamically generated @@ -851,8 +869,14 @@ public void renderSignature() throws IOException { @Override public void renderDocs() throws Exception { - // Currently we only render kibana definition files, but we could expand to rendering much more if we decide to - renderKibanaCommandDefinition(); + // Currently we only render either signatures or kibana definition files, + // but we could expand to rendering much more if we decide to + if (args != null) { + renderTypes(name, args); + } + if (licenseState != null) { + renderKibanaCommandDefinition(); + } } void renderKibanaCommandDefinition() throws Exception { @@ -872,6 +896,47 @@ void renderKibanaCommandDefinition() throws Exception { writeToTempKibanaDir("definition", "json", rendered); } } + + @Override + void renderTypes(String name, List args) throws IOException { + assert args.size() == 2; + StringBuilder header = new StringBuilder("| "); + StringBuilder separator = new StringBuilder("| "); + List argNames = args.stream().map(EsqlFunctionRegistry.ArgSignature::name).toList(); + for (String arg : argNames) { + header.append(arg).append(" | "); + separator.append("---").append(" | "); + } + + Map> compactedTable = new TreeMap<>(); + for (Map.Entry, DataType> sig : this.signatures.get().entrySet()) { + if (shouldHideSignature(sig.getKey(), sig.getValue())) { + continue; + } + String mainType = sig.getKey().getFirst().esNameIfPossible(); + String secondaryType = sig.getKey().get(1).esNameIfPossible(); + List secondaryTypes = compactedTable.computeIfAbsent(mainType, (k) -> new ArrayList<>()); + secondaryTypes.add(secondaryType); + } + + List table = new ArrayList<>(); + for (Map.Entry> sig : compactedTable.entrySet()) { + String row = "| " + sig.getKey() + " | " + String.join(", ", sig.getValue()) + " |"; + table.add(row); + } + Collections.sort(table); + if (table.isEmpty()) { + logger.info("Warning: No table of types generated for [{}]", name); + return; + } + + String rendered = DOCS_WARNING + """ + **Supported types** + + """ + header + "\n" + separator + "\n" + String.join("\n", table) + "\n\n"; + logger.info("Writing function types for [{}]:\n{}", name, rendered); + writeToTempSnippetsDir("types", rendered); + } } protected String buildFunctionSignatureSvg() throws IOException { @@ -893,6 +958,7 @@ void renderParametersList(List argNames, List argDescriptions) t } void renderTypes(String name, List args) throws IOException { + boolean showResultColumn = signatures.get().values().stream().anyMatch(Objects::nonNull); StringBuilder header = new StringBuilder("| "); StringBuilder separator = new StringBuilder("| "); List argNames = args.stream().map(EsqlFunctionRegistry.ArgSignature::name).toList(); @@ -900,8 +966,10 @@ void renderTypes(String name, List args) thro header.append(arg).append(" | "); separator.append("---").append(" | "); } - header.append("result |"); - separator.append("--- |"); + if (showResultColumn) { + header.append("result |"); + separator.append("--- |"); + } List table = new ArrayList<>(); for (Map.Entry, DataType> sig : this.signatures.get().entrySet()) { // TODO flip to using sortedSignatures @@ -911,7 +979,7 @@ void renderTypes(String name, List args) thro if (sig.getKey().size() > argNames.size()) { // skip variadic [test] cases (but not those with optional parameters) continue; } - table.add(getTypeRow(args, sig, argNames)); + table.add(getTypeRow(args, sig, argNames, showResultColumn)); } Collections.sort(table); if (table.isEmpty()) { @@ -930,7 +998,8 @@ void renderTypes(String name, List args) thro private static String getTypeRow( List args, Map.Entry, DataType> sig, - List argNames + List argNames, + boolean showResultColumn ) { StringBuilder b = new StringBuilder("| "); for (int i = 0; i < sig.getKey().size(); i++) { @@ -944,8 +1013,10 @@ private static String getTypeRow( b.append(" | "); } b.append("| ".repeat(argNames.size() - sig.getKey().size())); - b.append(sig.getValue().esNameIfPossible()); - b.append(" |"); + if (showResultColumn) { + b.append(sig.getValue().esNameIfPossible()); + b.append(" |"); + } return b.toString(); } @@ -1021,19 +1092,21 @@ void renderKibanaInlineDocs(String name, String titleName, FunctionInfo info) th builder.append("### ").append(titleName.toUpperCase(Locale.ROOT)).append("\n"); String cleanedDesc = replaceLinks(info.description()); cleanedDesc = removeAppliesToBlocks(cleanedDesc); - builder.append(cleanedDesc).append("\n\n"); + builder.append(cleanedDesc).append("\n"); + + if (Strings.isNullOrEmpty(info.note()) == false) { + String cleanedNote = replaceLinks(info.note()); + cleanedNote = removeAppliesToBlocks(cleanedNote); + builder.append("\nNote: ").append(cleanedNote).append("\n"); + } if (info.examples().length > 0) { Example example = info.examples()[0]; - builder.append("```esql\n"); + builder.append("\n```esql\n"); builder.append(loadExample(example.file(), example.tag())); builder.append("\n```\n"); } - if (Strings.isNullOrEmpty(info.note()) == false) { - String cleanedNote = replaceLinks(info.note()); - cleanedNote = removeAppliesToBlocks(cleanedNote); - builder.append("Note: ").append(cleanedNote).append("\n"); - } + String rendered = builder.toString(); logger.info("Writing kibana inline docs for [{}]:\n{}", name, rendered); writeToTempKibanaDir("docs", "md", rendered); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/NullPredicatesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/NullPredicatesTests.java deleted file mode 100644 index 69bfcc99a21ea..0000000000000 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/NullPredicatesTests.java +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.esql.expression.predicate.operator; - -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; -import org.elasticsearch.xpack.esql.expression.function.DocsV3Support; -import org.elasticsearch.xpack.esql.expression.function.Example; -import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; -import org.elasticsearch.xpack.esql.expression.function.Param; -import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToStringTests; -import org.junit.AfterClass; - -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -/** - * In the documentation we document `IS NULL` and `IS NOT NULL` together. - */ -public class NullPredicatesTests extends ESTestCase { - public void testDummy() { - assert true; - } - - @AfterClass - public static void renderDocs() throws Exception { - if (System.getProperty("generateDocs") == null) { - return; - } - renderNullPredicate( - new DocsV3Support.OperatorConfig( - "predicates", - "IS NULL and IS NOT NULL", - TestNullPredicates.class, - DocsV3Support.OperatorCategory.UNARY - ) - ); - renderNullPredicate( - new DocsV3Support.OperatorConfig( - "is_null", - "IS NULL", - TestIsNullPredicate.class, - DocsV3Support.OperatorCategory.NULL_PREDICATES - ) - ); - renderNullPredicate( - new DocsV3Support.OperatorConfig( - "is_not_null", - "IS NOT NULL", - TestIsNotNullPredicate.class, - DocsV3Support.OperatorCategory.NULL_PREDICATES - ) - ); - } - - private static void renderNullPredicate(DocsV3Support.OperatorConfig op) throws Exception { - var docs = new DocsV3Support.OperatorsDocsSupport(op.name(), NullPredicatesTests.class, op, NullPredicatesTests::signatures); - docs.renderSignature(); - docs.renderDocs(); - } - - public static Map, DataType> signatures() { - // TODO: Verify the correct datatypes for this - Map, DataType> toString = AbstractFunctionTestCase.signatures(ToStringTests.class); - Map, DataType> results = new LinkedHashMap<>(); - for (var entry : toString.entrySet()) { - DataType dataType = entry.getKey().getFirst(); - results.put(List.of(dataType), DataType.BOOLEAN); - } - return results; - } - - /** - * This class only exists to provide FunctionInfo for the documentation - */ - public class TestNullPredicates { - @FunctionInfo( - returnType = {}, - description = "For NULL comparison use the `IS NULL` and `IS NOT NULL` predicates.", - examples = { @Example(file = "null", tag = "is-null"), @Example(file = "null", tag = "is-not-null") } - ) - public TestNullPredicates( - @Param( - name = "field", - type = { - "boolean", - "cartesian_point", - "cartesian_shape", - "date", - "date_nanos", - "double", - "geo_point", - "geo_shape", - "integer", - "ip", - "keyword", - "long", - "text", - "unsigned_long", - "version" }, - description = "Input value. The input can be a single- or multi-valued column or an expression." - ) Expression v - ) {} - } - - /** - * This class only exists to provide FunctionInfo for the documentation - */ - public class TestIsNullPredicate { - @FunctionInfo( - operator = "IS NULL", - returnType = {}, - description = "Use `IS NULL` to filter data based on whether the field exists or not.", - examples = { @Example(file = "null", tag = "is-null") } - ) - public TestIsNullPredicate( - @Param( - name = "field", - type = { - "boolean", - "cartesian_point", - "cartesian_shape", - "date", - "date_nanos", - "double", - "geo_point", - "geo_shape", - "integer", - "ip", - "keyword", - "long", - "text", - "unsigned_long", - "version" }, - description = "Input value. The input can be a single- or multi-valued column or an expression." - ) Expression v - ) {} - } - - /** - * This class only exists to provide FunctionInfo for the documentation - */ - public class TestIsNotNullPredicate { - @FunctionInfo( - operator = "IS NOT NULL", - returnType = {}, - description = "Use `IS NOT NULL` to filter data based on whether the field exists or not.", - examples = { @Example(file = "null", tag = "is-not-null") } - ) - public TestIsNotNullPredicate( - @Param( - name = "field", - type = { - "boolean", - "cartesian_point", - "cartesian_shape", - "date", - "date_nanos", - "double", - "geo_point", - "geo_shape", - "integer", - "ip", - "keyword", - "long", - "text", - "unsigned_long", - "version" }, - description = "Input value. The input can be a single- or multi-valued column or an expression." - ) Expression v - ) {} - } -}