diff --git a/docs/extend/example-text-analysis-plugin.md b/docs/extend/example-text-analysis-plugin.md index 8943e39c356b0..82fbb49277f54 100644 --- a/docs/extend/example-text-analysis-plugin.md +++ b/docs/extend/example-text-analysis-plugin.md @@ -131,6 +131,7 @@ Elastic provides a Grade plugin, `elasticsearch.stable-esplugin`, that makes it "filter": ["hello_world"] } ``` + % TEST[skip:would require this plugin to be installed] @@ -162,6 +163,7 @@ If you are using the `elasticsearch.stable-esplugin` plugin for Gradle, you can } } ``` + % NOTCONSOLE 4. In `src/yamlRestTest/resources/rest-api-spec/test/plugin`, create the `10_token_filter.yml` YAML file: diff --git a/docs/reference/aggregations/search-aggregations-metrics-scripted-metric-aggregation.md b/docs/reference/aggregations/search-aggregations-metrics-scripted-metric-aggregation.md index 0a20a62b576fe..e380ac8fb1f07 100644 --- a/docs/reference/aggregations/search-aggregations-metrics-scripted-metric-aggregation.md +++ b/docs/reference/aggregations/search-aggregations-metrics-scripted-metric-aggregation.md @@ -168,19 +168,23 @@ Lets say that documents 1 and 3 end up on shard A and documents 2 and 4 end up o This is run once on each shard before any document collection is performed, and so we will have a copy on each shard: Shard A -: ```js -"state" : { - "transactions" : [] -} -``` +: + ```js + "state" : { + "transactions" : [] + } + ``` + % NOTCONSOLE Shard B -: ```js -"state" : { - "transactions" : [] -} -``` +: + ```js + "state" : { + "transactions" : [] + } + ``` + % NOTCONSOLE @@ -189,19 +193,23 @@ Shard B Each shard collects its documents and runs the map_script on each document that is collected: Shard A -: ```js -"state" : { - "transactions" : [ 80, -30 ] -} -``` +: + ```js + "state" : { + "transactions" : [ 80, -30 ] + } + ``` + % NOTCONSOLE Shard B -: ```js -"state" : { - "transactions" : [ -10, 130 ] -} -``` +: + ```js + "state" : { + "transactions" : [ -10, 130 ] + } + ``` + % NOTCONSOLE diff --git a/docs/reference/elasticsearch/configuration-reference/cluster-level-shard-allocation-routing-settings.md b/docs/reference/elasticsearch/configuration-reference/cluster-level-shard-allocation-routing-settings.md index 47a7e15c83932..7b2553c5696cc 100644 --- a/docs/reference/elasticsearch/configuration-reference/cluster-level-shard-allocation-routing-settings.md +++ b/docs/reference/elasticsearch/configuration-reference/cluster-level-shard-allocation-routing-settings.md @@ -254,6 +254,7 @@ PUT _cluster/settings } } ``` +% TEST[skip:TODO] ### Cluster routing settings [cluster-routing-settings] @@ -304,7 +305,7 @@ PUT _cluster/settings } } ``` - +% TEST[skip:TODO] ## Node allocation stats cache [node-allocation-stats-cache] diff --git a/docs/reference/elasticsearch/elasticsearch-audit-events.md b/docs/reference/elasticsearch/elasticsearch-audit-events.md index 745a7b9935bd4..37022f774dca6 100644 --- a/docs/reference/elasticsearch/elasticsearch-audit-events.md +++ b/docs/reference/elasticsearch/elasticsearch-audit-events.md @@ -43,6 +43,7 @@ $$$event-access-denied$$$ "action":"indices:admin/auto_create", "request.name":"CreateIndexRequest", "indices":[""]} ``` + % NOTCONSOLE :::: @@ -62,6 +63,7 @@ $$$event-access-granted$$$ "origin.address":"[::1]:52434", "request.id":"yKOgWn2CRQCKYgZRz3phJw", "action":"indices:data/write/bulk", "request.name":"BulkRequest"} ``` + % NOTCONSOLE :::: @@ -79,6 +81,7 @@ $$$event-anonymous-access-denied$$$ "[::1]:50543", "url.path":"/twitter/_async_search", "url.query":"pretty", "request.method":"POST", "request.id":"TqA9OisyQ8WTl1ivJUV1AA"} ``` + % NOTCONSOLE :::: @@ -97,6 +100,7 @@ $$$event-authentication-failed$$$ "url.query":"pretty", "request.method":"POST", "request.id":"POv8p_qeTl2tb5xoFl0HIg"} ``` + % NOTCONSOLE :::: @@ -116,6 +120,7 @@ $$$event-authentication-success$$$ "url.query":"pretty", "request.method":"POST", "request.id":"nHV3UMOoSiu-TaSPWCfxGg"} ``` + % NOTCONSOLE :::: @@ -134,6 +139,7 @@ $$$event-change-disable-user$$$ action":"change_disable_user", "request.id":"qvLIgw_eTvyK3cgV-GaLVg", "change":{"disable":{"user":{"name":"user1"}}}} ``` + % NOTCONSOLE :::: @@ -152,6 +158,7 @@ $$$event-change-enable-user$$$ action":"change_enable_user", "request.id":"BO3QU3qeTb-Ei0G0rUOalQ", "change":{"enable":{"user":{"name":"user1"}}}} ``` + % NOTCONSOLE :::: @@ -170,6 +177,7 @@ $$$event-change-password$$$ action":"change_password", "request.id":"bz5a1Cc3RrebDMitMGGNCw", "change":{"password":{"user":{"name":"user1"}}}} ``` + % NOTCONSOLE :::: @@ -188,6 +196,7 @@ $$$event-create-service-token$$$ action":"create_service_token", "request.id":"az9a1Db5QrebDMacQ8yGKc", "create":{"service_token":{"namespace":"elastic","service":"fleet-server","name":"token1"}}}` ``` + % NOTCONSOLE :::: @@ -204,6 +213,7 @@ $$$event-connection-denied$$$ "connection_denied", "origin.type":"rest", "origin.address":"10.10.0.20:52314", "transport.profile":".http", "rule":"deny 10.10.0.0/16"} ``` + % NOTCONSOLE :::: @@ -220,6 +230,7 @@ $$$event-connection-granted$$$ "connection_granted", "origin.type":"rest", "origin.address":"[::1]:52314", "transport.profile":".http", "rule":"allow ::1,127.0.0.1"} ``` + % NOTCONSOLE :::: @@ -245,6 +256,7 @@ $$$event-create-apikey$$$ "metadata":{"application":"my-application","environment":{"level": 1, "tags":["dev","staging"]}}}}} ``` + % NOTCONSOLE :::: @@ -270,6 +282,7 @@ $$$event-change-apikey$$$ "metadata":{"application":"my-application","environment":{"level": 1, "tags":["dev","staging"]}},"expiration":"10d"}}} ``` + % NOTCONSOLE :::: @@ -296,6 +309,7 @@ $$$event-change-apikeys$$$ "metadata":{"application":"my-application","environment":{"level":1, "tags":["dev","staging"]}},"expiration":"10d"}}} ``` + % NOTCONSOLE :::: @@ -314,6 +328,7 @@ $$$event-delete-privileges$$$ action":"delete_privileges", "request.id":"7wRWVxxqTzCKEspeSP7J8g", "delete":{"privileges":{"application":"myapp","privileges":["read"]}}} ``` + % NOTCONSOLE :::: @@ -332,6 +347,7 @@ $$$event-delete-role$$$ "delete_role", "request.id":"155IKq3zQdWq-12dgKZRnw", "delete":{"role":{"name":"my_admin_role"}}} ``` + % NOTCONSOLE :::: @@ -350,6 +366,7 @@ $$$event-delete-role-mapping$$$ action":"delete_role_mapping", "request.id":"Stim-DuoSTCWom0S_xhf8g", "delete":{"role_mapping":{"name":"mapping1"}}} ``` + % NOTCONSOLE :::: @@ -368,6 +385,7 @@ $$$event-delete-service-token$$$ action":"delete_service_token", "request.id":"az9a1Db5QrebDMacQ8yGKc", "delete":{"service_token":{"namespace":"elastic","service":"fleet-server","name":"token1"}}} ``` + % NOTCONSOLE :::: @@ -386,6 +404,7 @@ $$$event-delete-user$$$ "event.action":"delete_user", "request.id":"au5a1Cc3RrebDMitMGGNCw", "delete":{"user":{"name":"jacknich"}}} ``` + % NOTCONSOLE :::: @@ -405,6 +424,7 @@ $$$event-invalidate-apikeys$$$ "invalidate":{"apikeys":{"owned_by_authenticated_user":false, "user":{"name":"myuser","realm":"native1"}}}} ``` + % NOTCONSOLE :::: @@ -424,6 +444,7 @@ $$$event-put-privileges$$$ "put":{"privileges":[{"application":"myapp","name":"read","actions": ["data:read/*","action:login"],"metadata":{"description":"Read access to myapp"}}]}} ``` + % NOTCONSOLE :::: @@ -446,6 +467,7 @@ $$$event-put-role$$$ {"names":["apm-all*"],"privileges":["all"],"query":"{\"term\": {\"service.name\": \"bar2\"}}"}],"applications":[],"run_as":[]}}}} ``` + % NOTCONSOLE :::: @@ -465,6 +487,7 @@ $$$event-put-role-mapping$$$ "put":{"role_mapping":{"name":"mapping1","roles":["user"],"rules": {"field":{"username":"*"}},"enabled":true,"metadata":{"version":1}}}} ``` + % NOTCONSOLE :::: @@ -485,6 +508,7 @@ $$$event-put-user$$$ "full_name":"Jack Sparrow","email":"jack@blackpearl.com", "has_password":true,"metadata":{"cunning":10}}}} ``` + % NOTCONSOLE :::: @@ -503,6 +527,7 @@ $$$event-realm-auth-failed$$$ "/_security/user/user1", "url.query":"pretty", "request.method":"POST", "request.id":"POv8p_qeTl2tb5xoFl0HIg"} ``` + % NOTCONSOLE :::: @@ -522,6 +547,7 @@ $$$event-runas-denied$$$ "[::1]:52662", "request.id":"RcaSt872RG-R_WJBEGfYXA", "action":"indices:data/read/search", "request.name":"SearchRequest", "indices":["alias1"]} ``` + % NOTCONSOLE :::: @@ -541,6 +567,7 @@ $$$event-runas-granted$$$ "[::1]:52623", "request.id":"dGqPTdEQSX2TAPS3cvc1qA", "action": "indices:data/read/search", "request.name":"SearchRequest", "indices":["alias1"]} ``` + % NOTCONSOLE :::: @@ -568,6 +595,7 @@ $$$event-tampered-request$$$ "/twitter/_async_search", "url.query":"pretty", "request.method":"POST", "request.id":"TqA9OisyQ8WTl1ivJUV1AA"} ``` + % NOTCONSOLE :::: diff --git a/docs/reference/elasticsearch/index-settings/index-modules.md b/docs/reference/elasticsearch/index-settings/index-modules.md index 8e76f58030686..708578b023c1a 100644 --- a/docs/reference/elasticsearch/index-settings/index-modules.md +++ b/docs/reference/elasticsearch/index-settings/index-modules.md @@ -62,11 +62,12 @@ $$$index-mode-setting$$$ `index.mode` {applies_to}`serverless: all` { "settings": { "index":{ - "mode":"standard" # This index uses the `standard` index mode + "mode":"standard" <1> } } } ``` + 1. This index uses the `standard` index mode **Supported values** The `index.mode` setting supports the following values: diff --git a/docs/reference/elasticsearch/index-settings/slow-log.md b/docs/reference/elasticsearch/index-settings/slow-log.md index e3a054d8e807a..c8caa60035bbe 100644 --- a/docs/reference/elasticsearch/index-settings/slow-log.md +++ b/docs/reference/elasticsearch/index-settings/slow-log.md @@ -209,6 +209,7 @@ If you aren’t sure how to start investigating traffic issues, consider enablin "index.search.slowlog.threshold.query.warn": "30s" } ``` + % TEST[setup:my_index] * Enable for indexing requests: @@ -219,6 +220,7 @@ If you aren’t sure how to start investigating traffic issues, consider enablin "index.indexing.slowlog.threshold.index.warn": "30s" } ``` + % TEST[setup:my_index] Slow log thresholds being met does not guarantee cluster performance issues. In the event that symptoms are noticed, slow logs can provide helpful data to diagnose upstream traffic patterns or sources to resolve client-side issues. For example, you can use data included in `X-Opaque-ID`, the `_source` request body, or `user.*` fields to identify the source of your issue. This is similar to troubleshooting [live expensive tasks](docs-content://troubleshoot/elasticsearch/task-queue-backlog.md). diff --git a/docs/reference/elasticsearch/mapping-reference/dense-vector.md b/docs/reference/elasticsearch/mapping-reference/dense-vector.md index bff2cc534affa..6f64d843f44ef 100644 --- a/docs/reference/elasticsearch/mapping-reference/dense-vector.md +++ b/docs/reference/elasticsearch/mapping-reference/dense-vector.md @@ -121,6 +121,7 @@ To retrieve vector values explicitly, you can use: "fields": ["my_vector"] } ``` + % TEST[continued] - The `_source.exclude_vectors` flag to re-enable vector inclusion in `_source` responses: @@ -132,6 +133,7 @@ To retrieve vector values explicitly, you can use: } } ``` + % TEST[continued] :::{tip} For more context about the decision to exclude vectors from `_source` by default, read the [blog post](https://www.elastic.co/search-labs/blog/elasticsearch-exclude-vectors-from-source). @@ -468,32 +470,32 @@ POST /my-bit-vectors/_search?filter_path=hits.hits ```console-result { - "hits": { - "hits": [ - { - "_index": "my-bit-vectors", - "_id": "1", - "_score": 1.0, - "_source": { - "my_vector": [ - 127, - -127, - 0, - 1, - 42 - ] - } - }, - { - "_index": "my-bit-vectors", - "_id": "2", - "_score": 0.55, - "_source": { - "my_vector": "8100012a7f" - } - } - ] - } + "hits": { + "hits": [ + { + "_index": "my-bit-vectors", + "_id": "1", + "_score": 1, + "_source": { + "my_vector": [ + 127, + -127, + 0, + 1, + 42 + ] + } + }, + { + "_index": "my-bit-vectors", + "_id": "2", + "_score": 0.55, + "_source": { + "my_vector": "8100012a7f" + } + } + ] + } } ``` diff --git a/docs/reference/elasticsearch/mapping-reference/sparse-vector.md b/docs/reference/elasticsearch/mapping-reference/sparse-vector.md index e6741b969541a..24793ee51da4d 100644 --- a/docs/reference/elasticsearch/mapping-reference/sparse-vector.md +++ b/docs/reference/elasticsearch/mapping-reference/sparse-vector.md @@ -114,6 +114,7 @@ POST my-index-2/_search "fields": ["my_vector"] } ``` +% TEST[skip:no index setup] - The `_source.exclude_vectors` flag to re-enable vector inclusion in `_source` responses: @@ -125,6 +126,7 @@ POST my-index-2/_search } } ``` +% TEST[skip:no index setup] :::{tip} For more context about the decision to exclude vectors from `_source` by default, read the [blog post](https://www.elastic.co/search-labs/blog/elasticsearch-exclude-vectors-from-source). @@ -265,10 +267,11 @@ POST /my-index/_update/1 } } ``` +% TEST[continued] Observe that tokens are merged, not replaced: -```json +```js { "my_vector": { "token_a": 0.5, @@ -277,6 +280,7 @@ Observe that tokens are merged, not replaced: } } ``` +% NOTCONSOLE ### Replacing the entire `sparse_vector` field @@ -296,6 +300,7 @@ POST /my-index/_update/1 } } ``` +% TEST[continued] :::{note} This same merging behavior also applies to [`rank_features` fields](/reference/elasticsearch/mapping-reference/rank-features.md), because they are also object-like structures. diff --git a/docs/reference/elasticsearch/rest-apis/retrieve-inner-hits.md b/docs/reference/elasticsearch/rest-apis/retrieve-inner-hits.md index e966ba7a10229..58551a3570947 100644 --- a/docs/reference/elasticsearch/rest-apis/retrieve-inner-hits.md +++ b/docs/reference/elasticsearch/rest-apis/retrieve-inner-hits.md @@ -175,8 +175,8 @@ An example of a response snippet that could be generated from the above search r } } ``` -% TESTRESPONSE[s/"_source": \.\.\./"_source": $body.hits.hits.0._source/] -% TESTRESPONSE[s/\.\.\./"timed_out": false, "took": $body.took, "_shards": $body._shards/] +% TESTRESPONSE[s/"_source": \.\.\./"_source": $body.hits.hits.0._source/] +% TESTRESPONSE[s/\.\.\./"timed_out": false, "took": $body.took, "_shards": $body._shards/] 1. The name used in the inner hit definition in the search request. A custom key can be used via the `name` option. diff --git a/docs/reference/elasticsearch/rest-apis/retrievers.md b/docs/reference/elasticsearch/rest-apis/retrievers.md index d797964043355..d19386f179fc5 100644 --- a/docs/reference/elasticsearch/rest-apis/retrievers.md +++ b/docs/reference/elasticsearch/rest-apis/retrievers.md @@ -95,6 +95,88 @@ If you use the `none` normalizer, the scores across field groups will not be nor When using the `linear` retriever, fields can be boosted using the `^` notation: + + ```console GET books/_search { @@ -145,6 +227,7 @@ PUT /books } } ``` +% TEST[continued] And we run this query: @@ -165,6 +248,7 @@ GET books/_search } } ``` +% TEST[continued] The score breakdown would be: @@ -194,6 +278,7 @@ GET books/_search } } ``` +% TEST[continued] The score breakdown would change to: @@ -222,6 +307,7 @@ GET books/_search } } ``` +% TEST[continued] 1. Match fields that start with `title` 2. Match fields that end with `_text` diff --git a/docs/reference/elasticsearch/rest-apis/retrievers/knn-retriever.md b/docs/reference/elasticsearch/rest-apis/retrievers/knn-retriever.md index 20ab33cb82de6..ef2c28f331722 100644 --- a/docs/reference/elasticsearch/rest-apis/retrievers/knn-retriever.md +++ b/docs/reference/elasticsearch/rest-apis/retrievers/knn-retriever.md @@ -109,6 +109,7 @@ PUT /restaurants } } } + POST /restaurants/_bulk?refresh {"index":{}} {"region": "Austria", "year": "2019", "vector": [10, 22, 77]} @@ -118,7 +119,9 @@ POST /restaurants/_bulk?refresh {"region": "Austria", "year": "2020", "vector": [10, 22, 79]} {"index":{}} {"region": "France", "year": "2020", "vector": [10, 22, 80]} + PUT /movies + PUT /books { "mappings": { @@ -140,6 +143,7 @@ PUT /books } } } + PUT _query_rules/my-ruleset { "rules": [ @@ -163,6 +167,7 @@ PUT _query_rules/my-ruleset } ``` % TESTSETUP + ```console DELETE /restaurants DELETE /movies diff --git a/docs/reference/elasticsearch/rest-apis/retrievers/linear-retriever.md b/docs/reference/elasticsearch/rest-apis/retrievers/linear-retriever.md index 296c8f311a85a..36da0f5104df4 100644 --- a/docs/reference/elasticsearch/rest-apis/retrievers/linear-retriever.md +++ b/docs/reference/elasticsearch/rest-apis/retrievers/linear-retriever.md @@ -135,6 +135,7 @@ GET my_index/_search } } ``` + 1. KNN query weighted 5x 2. BM25 query weighted 1.5x diff --git a/docs/reference/elasticsearch/rest-apis/retrievers/pinned-retriever.md b/docs/reference/elasticsearch/rest-apis/retrievers/pinned-retriever.md index 0be4ec08f8494..503de03824ca5 100644 --- a/docs/reference/elasticsearch/rest-apis/retrievers/pinned-retriever.md +++ b/docs/reference/elasticsearch/rest-apis/retrievers/pinned-retriever.md @@ -46,6 +46,7 @@ PUT /restaurants } } } + POST /restaurants/_bulk?refresh {"index":{}} {"region": "Austria", "year": "2019", "vector": [10, 22, 77]} @@ -55,7 +56,9 @@ POST /restaurants/_bulk?refresh {"region": "Austria", "year": "2020", "vector": [10, 22, 79]} {"index":{}} {"region": "France", "year": "2020", "vector": [10, 22, 80]} + PUT /movies + PUT /books { "mappings": { @@ -77,6 +80,7 @@ PUT /books } } } + PUT _query_rules/my-ruleset { "rules": [ @@ -100,6 +104,7 @@ PUT _query_rules/my-ruleset } ``` % TESTSETUP + ```console DELETE /restaurants DELETE /movies diff --git a/docs/reference/elasticsearch/rest-apis/retrievers/rescorer-retriever.md b/docs/reference/elasticsearch/rest-apis/retrievers/rescorer-retriever.md index d6a5393b051e1..2bee83255555e 100644 --- a/docs/reference/elasticsearch/rest-apis/retrievers/rescorer-retriever.md +++ b/docs/reference/elasticsearch/rest-apis/retrievers/rescorer-retriever.md @@ -61,6 +61,7 @@ PUT /restaurants } } } + POST /restaurants/_bulk?refresh {"index":{}} {"region": "Austria", "year": "2019", "vector": [10, 22, 77]} @@ -70,7 +71,9 @@ POST /restaurants/_bulk?refresh {"region": "Austria", "year": "2020", "vector": [10, 22, 79]} {"index":{}} {"region": "France", "year": "2020", "vector": [10, 22, 80]} + PUT /movies + PUT /books { "mappings": { @@ -92,6 +95,7 @@ PUT /books } } } + PUT _query_rules/my-ruleset { "rules": [ @@ -115,6 +119,7 @@ PUT _query_rules/my-ruleset } ``` % TESTSETUP + ```console DELETE /restaurants DELETE /movies diff --git a/docs/reference/elasticsearch/rest-apis/retrievers/rrf-retriever.md b/docs/reference/elasticsearch/rest-apis/retrievers/rrf-retriever.md index 55dfbe528e41a..36c7ce4cad453 100644 --- a/docs/reference/elasticsearch/rest-apis/retrievers/rrf-retriever.md +++ b/docs/reference/elasticsearch/rest-apis/retrievers/rrf-retriever.md @@ -72,6 +72,7 @@ PUT /restaurants } } } + POST /restaurants/_bulk?refresh {"index":{}} {"region": "Austria", "year": "2019", "vector": [10, 22, 77]} @@ -81,7 +82,9 @@ POST /restaurants/_bulk?refresh {"region": "Austria", "year": "2020", "vector": [10, 22, 79]} {"index":{}} {"region": "France", "year": "2020", "vector": [10, 22, 80]} + PUT /movies + PUT /books { "mappings": { @@ -103,6 +106,7 @@ PUT /books } } } + PUT _query_rules/my-ruleset { "rules": [ @@ -126,6 +130,7 @@ PUT _query_rules/my-ruleset } ``` % TESTSETUP + ```console DELETE /restaurants DELETE /movies diff --git a/docs/reference/elasticsearch/rest-apis/retrievers/rule-retriever.md b/docs/reference/elasticsearch/rest-apis/retrievers/rule-retriever.md index c0b97a15d707d..051e652b012f6 100644 --- a/docs/reference/elasticsearch/rest-apis/retrievers/rule-retriever.md +++ b/docs/reference/elasticsearch/rest-apis/retrievers/rule-retriever.md @@ -56,6 +56,7 @@ PUT /restaurants } } } + POST /restaurants/_bulk?refresh {"index":{}} {"region": "Austria", "year": "2019", "vector": [10, 22, 77]} @@ -65,7 +66,9 @@ POST /restaurants/_bulk?refresh {"region": "Austria", "year": "2020", "vector": [10, 22, 79]} {"index":{}} {"region": "France", "year": "2020", "vector": [10, 22, 80]} + PUT /movies + PUT /books { "mappings": { @@ -87,6 +90,7 @@ PUT /books } } } + PUT _query_rules/my-ruleset { "rules": [ diff --git a/docs/reference/elasticsearch/rest-apis/retrievers/standard-retriever.md b/docs/reference/elasticsearch/rest-apis/retrievers/standard-retriever.md index 7f88e408adee3..4bdadbd7b06d5 100644 --- a/docs/reference/elasticsearch/rest-apis/retrievers/standard-retriever.md +++ b/docs/reference/elasticsearch/rest-apis/retrievers/standard-retriever.md @@ -71,6 +71,7 @@ PUT /restaurants } } } + POST /restaurants/_bulk?refresh {"index":{}} {"region": "Austria", "year": "2019", "vector": [10, 22, 77]} @@ -80,7 +81,9 @@ POST /restaurants/_bulk?refresh {"region": "Austria", "year": "2020", "vector": [10, 22, 79]} {"index":{}} {"region": "France", "year": "2020", "vector": [10, 22, 80]} + PUT /movies + PUT /books { "mappings": { @@ -102,6 +105,7 @@ PUT /books } } } + PUT _query_rules/my-ruleset { "rules": [ @@ -125,6 +129,7 @@ PUT _query_rules/my-ruleset } ``` % TESTSETUP + ```console DELETE /restaurants DELETE /movies @@ -162,7 +167,6 @@ GET /restaurants/_search } } ``` -% TEST[continued] 1. Opens the `retriever` object. 2. The `standard` retriever is used for defining traditional {{es}} queries. diff --git a/docs/reference/elasticsearch/rest-apis/retrievers/text-similarity-reranker-retriever.md b/docs/reference/elasticsearch/rest-apis/retrievers/text-similarity-reranker-retriever.md index 5c9f8cb777049..a15e61ac3e605 100644 --- a/docs/reference/elasticsearch/rest-apis/retrievers/text-similarity-reranker-retriever.md +++ b/docs/reference/elasticsearch/rest-apis/retrievers/text-similarity-reranker-retriever.md @@ -117,6 +117,87 @@ Refer to this [Python notebook](https://github.com/elastic/elasticsearch-labs/bl This example demonstrates how to deploy the [Elastic Rerank](docs-content://explore-analyze/machine-learning/nlp/ml-nlp-rerank.md) model and use it to re-rank search results using the `text_similarity_reranker` retriever. + + Follow these steps: 1. Create an inference endpoint for the `rerank` task using the [Create {{infer}} API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put). @@ -216,6 +297,7 @@ Follow these steps to load the model and create a semantic re-ranker. ```sh python -m pip install eland[pytorch] ``` + % NOTCONSOLE 2. Upload the model to {{es}} using Eland. This example assumes you have an Elastic Cloud deployment and an API key. Refer to the [Eland documentation](eland://reference/machine-learning.md#ml-nlp-pytorch-auth) for more authentication options. @@ -228,6 +310,7 @@ Follow these steps to load the model and create a semantic re-ranker. --clear-previous \ --start ``` + % NOTCONSOLE 3. Create an inference endpoint for the `rerank` task diff --git a/docs/reference/elasticsearch/rest-apis/search-profile.md b/docs/reference/elasticsearch/rest-apis/search-profile.md index 76e57c9c7de30..c31ee1e7e38ff 100644 --- a/docs/reference/elasticsearch/rest-apis/search-profile.md +++ b/docs/reference/elasticsearch/rest-apis/search-profile.md @@ -231,7 +231,7 @@ The API returns the following result: } ``` % TESTRESPONSE[s/"took": 25/"took": $body.took/] -% TESTRESPONSE[s/"hits": \[...\]/"hits": $body.$_path/] +% TESTRESPONSE[s/"hits": \[\.\.\.\]/"hits": $body.$_path/] % TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?/$body.$_path/] % TESTRESPONSE[s/"id": "\[q2aE02wS1R8qQFnYu6vDVQ\]\[my-index-000001\]\[0\]"/"id": $body.profile.shards.0.id/] % TESTRESPONSE[s/"node_id": "q2aE02wS1R8qQFnYu6vDVQ",/"node_id": "$body.profile.shards.0.node_id",/] @@ -267,14 +267,14 @@ The overall structure of the profile response is as follows: } } ``` -% TESTRESPONSE[s/"profile": /"took": $body.took, "timed_out": $body.timed_out, "_shards": $body._shards, "hits": $body.hits, "profile": /] -% TESTRESPONSE[s/(?>=[" ])\d+(\.\d+)?/$body.$_path/] -% TESTRESPONSE[s/"id": "\[q2aE02wS1R8qQFnYu6vDVQ\]\[my-index-000001\]\[0\]"/"id": $body.profile.shards.0.id/] -% TESTRESPONSE[s/"node_id": "q2aE02wS1R8qQFnYu6vDVQ",/"node_id": "$body.profile.shards.0.node_id",/] -% TESTRESPONSE[s/"query": \[…​\]/"query": $body.$_path/] -% TESTRESPONSE[s/"collector": \[…​\]/"collector": $body.$_path/] -% TESTRESPONSE[s/"aggregations": \[…​\]/"aggregations": []/] -% TESTRESPONSE[s/"fetch": \{…​\}/"fetch": $body.$_path/] +% TESTRESPONSE[s/"profile": /"took": $body.took, "timed_out": $body.timed_out, "_shards": $body._shards, "hits": $body.hits, "profile": /] +% TESTRESPONSE[s/(?>=[" ])\d+(\.\d+)?/$body.$_path/] +% TESTRESPONSE[s/"id": "\[q2aE02wS1R8qQFnYu6vDVQ\]\[my-index-000001\]\[0\]"/"id": $body.profile.shards.0.id/] +% TESTRESPONSE[s/"node_id": "q2aE02wS1R8qQFnYu6vDVQ",/"node_id": "$body.profile.shards.0.node_id",/] +% TESTRESPONSE[s/"query": \[\.\.\.\]/"query": $body.$_path/] +% TESTRESPONSE[s/"collector": \[\.\.\.​\]/"collector": $body.$_path/] +% TESTRESPONSE[s/"aggregations": \[\.\.\.\]/"aggregations": []/] +% TESTRESPONSE[s/"fetch": \{\.\.\.​\}/"fetch": $body.$_path/] 1. A profile is returned for each shard that participated in the response, and is identified by a unique ID. 2. If the query was run on the local cluster, the cluster name is left out of the composite id and is marked "(local)" here. For a profile running on a remote_cluster using cross-cluster search, the "id" value would be something like `[q2aE02wS1R8qQFnYu6vDVQ][remote1:my-index-000001][0]` and the "cluster" value would be `remote1`. @@ -335,10 +335,10 @@ The `query` section contains detailed timing of the query tree executed by Lucen } ] ``` -% TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.profile.shards.0.id",\n"node_id": "$body.profile.shards.0.node_id",\n"shard_id": $body.profile.shards.0.shard_id,\n"index": "$body.profile.shards.0.index",\n"cluster": "(local)",\n"searches": [{\n/] -% TESTRESPONSE[s/]$/],"rewrite_time": $body.$_path, "collector": $body.$_path}], "aggregations": [], "fetch": $body.$_path}]}}/] -% TESTRESPONSE[s/(?>=[" ])\d+(\.\d+)?/$body.$_path/] -% TESTRESPONSE[s/"breakdown": \{…​\}/"breakdown": $body.$_path/] +% TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.profile.shards.0.id",\n"node_id": "$body.profile.shards.0.node_id",\n"shard_id": $body.profile.shards.0.shard_id,\n"index": "$body.profile.shards.0.index",\n"cluster": "(local)",\n"searches": [{\n/] +% TESTRESPONSE[s/]$/],"rewrite_time": $body.$_path, "collector": $body.$_path}], "aggregations": [], "fetch": $body.$_path}]}}/] +% TESTRESPONSE[s/(?>=[" ])\d+(\.\d+)?/$body.$_path/] +% TESTRESPONSE[s/"breakdown": \{\.\.\.​\}/"breakdown": $body.$_path/] 1. The breakdown timings are omitted for simplicity. @@ -378,9 +378,9 @@ The `breakdown` component lists detailed timing statistics about low-level Lucen "count_weight_count": 0 } ``` -% TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.profile.shards.0.id",\n"node_id": "$body.profile.shards.0.node_id",\n"shard_id": $body.profile.shards.0.shard_id,\n"index": "$body.profile.shards.0.index",\n"cluster": "(local)",\n"searches": [{\n"query": [{\n"type": "BooleanQuery",\n"description": "message:get message:search",\n"time_in_nanos": $body.$_path,/] -% TESTRESPONSE[s/}$/},\n"children": $body.$_path}],\n"rewrite_time": $body.$_path, "collector": $body.$_path}], "aggregations": [], "fetch": $body.$_path}]}}/] -% TESTRESPONSE[s/(?>=[" ])\d+(\.\d+)?/$body.$_path/] +% TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.profile.shards.0.id",\n"node_id": "$body.profile.shards.0.node_id",\n"shard_id": $body.profile.shards.0.shard_id,\n"index": "$body.profile.shards.0.index",\n"cluster": "(local)",\n"searches": [{\n"query": [{\n"type": "BooleanQuery",\n"description": "message:get message:search",\n"time_in_nanos": $body.$_path,/] +% TESTRESPONSE[s/}$/},\n"children": $body.$_path}],\n"rewrite_time": $body.$_path, "collector": $body.$_path}], "aggregations": [], "fetch": $body.$_path}]}}/] +% TESTRESPONSE[s/(?>=[" ])\d+(\.\d+)?/$body.$_path/] Timings are listed in wall-clock nanoseconds and are not normalized at all. All caveats about the overall `time_in_nanos` apply here. The intention of the breakdown is to give you a feel for A) what machinery in Lucene is actually eating time, and B) the magnitude of differences in times between the various components. Like the overall time, the breakdown is inclusive of all children times. @@ -508,8 +508,8 @@ GET /my-index-000001/_search } } ``` -% TEST[setup:my_index] -% TEST[s/_search/_search\?filter_path=profile.shards.id,profile.shards.node_id,profile.shards.shard_id,profile.shards.index,profile.shards.cluster,profile.shards.searches,profile.shards.aggregations,profile.shards.fetch/] +% TEST[setup:my_index] +% TEST[s/_search/_search\?filter_path=profile.shards.id,profile.shards.node_id,profile.shards.shard_id,profile.shards.index,profile.shards.cluster,profile.shards.searches,profile.shards.aggregations,profile.shards.fetch/] This example has: @@ -688,8 +688,8 @@ GET /my-index-000001/_search } } ``` -% TEST[s/_search/_search\?filter_path=profile.shards.aggregations/] -% TEST[continued] +% TEST[s/_search/_search\?filter_path=profile.shards.aggregations/] +% TEST[continued] This yields the following aggregation profile output: @@ -777,9 +777,9 @@ This yields the following aggregation profile output: } } ``` -% TESTRESPONSE[s/\.\.\.//] -% TESTRESPONSE[s/(?>=[" ])\d+(\.\d+)?/$body.$_path/] -% TESTRESPONSE[s/"id": "\[P6-vulHtQRWuD4YnubWb7A\]\[my-index-000001\]\[0\]"/"id": $body.profile.shards.0.id/] +% TESTRESPONSE[s/\.\.\.//] +% TESTRESPONSE[s/(?>=[" ])\d+(\.\d+)?/$body.$_path/] +% TESTRESPONSE[s/"id": "\[P6-vulHtQRWuD4YnubWb7A\]\[my-index-000001\]\[0\]"/"id": $body.profile.shards.0.id/] From the profile structure we can see that the `my_scoped_agg` is internally being run as a `NumericTermsAggregator` (because the field it is aggregating, `http.response.status_code`, is a numeric field). At the same level, we see a `GlobalAggregator` which comes from `my_global_agg`. That aggregation then has a child `NumericTermsAggregator` which comes from the second term’s aggregation on `http.response.status_code`. @@ -996,9 +996,9 @@ In the response, we see a profile which includes a `dfs` section for each shard } } ``` -% TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ "$body.$_path", {\n"id": "$body.$_path",\n"node_id": "$body.$_path",\n"shard_id": "$body.$_path",\n"index": "$body.$_path",\n"cluster": "$body.$_path",\n/] -% TESTRESPONSE[s/}$/}, "aggregations": [], "searches": $body.$_path}]}}/] -% TESTRESPONSE[s/(\-)?[0-9]+/ $body.$_path/] +% TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ "$body.$_path", {\n"id": "$body.$_path",\n"node_id": "$body.$_path",\n"shard_id": "$body.$_path",\n"index": "$body.$_path",\n"cluster": "$body.$_path",\n/] +% TESTRESPONSE[s/}$/}, "aggregations": [], "searches": $body.$_path}]}}/] +% TESTRESPONSE[s/(\-)?[0-9]+/ $body.$_path/] In the `dfs.statistics` portion of this response we see a `time_in_nanos` which is the total time it took to collect term statistics for this shard along with a further breakdown of the individual parts. diff --git a/docs/reference/elasticsearch/rest-apis/search-suggesters.md b/docs/reference/elasticsearch/rest-apis/search-suggesters.md index 1ae7637ba7367..3f8c9d1913b42 100644 --- a/docs/reference/elasticsearch/rest-apis/search-suggesters.md +++ b/docs/reference/elasticsearch/rest-apis/search-suggesters.md @@ -34,8 +34,8 @@ POST _search } } ``` -% TEST[setup:messages] -% TEST[s/^/PUT my-index-000001\/_mapping\n{"properties":{"user":{"properties":{"id":{"type":"keyword"}}}}}\n/] +% TEST[setup:messages] +% TEST[s/^/PUT my-index-000001\/_mapping\n{"properties":{"user":{"properties":{"id":{"type":"keyword"}}}}}\n/] The following suggest response example includes the suggestion response for `my-suggest-1` and `my-suggest-2`. Each suggestion part contains entries. Each entry is effectively a token from the suggest text and contains the suggestion entry text, the original start offset and length in the suggest text and if found an arbitrary number of options. @@ -67,7 +67,7 @@ The following suggest response example includes the suggestion response for `my- } ``` % TESTRESPONSE[s/"_shards": \.\.\./"_shards": "$body._shards",/] -% TESTRESPONSE[s/"hits": .../"hits": "$body.hits",/] +% TESTRESPONSE[s/"hits": \.\.\./"hits": "$body.hits",/] % TESTRESPONSE[s/"took": 2,/"took": "$body.took",/] % TESTRESPONSE[s/"my-suggest-2": \.\.\./"my-suggest-2": "$body.suggest.my-suggest-2"/] @@ -275,9 +275,9 @@ The response contains suggestions scored by the most likely spelling correction } } ``` -% TESTRESPONSE[s/"_shards": …​/"_shards": "$body._shards",/] -% TESTRESPONSE[s/"hits": …​/"hits": "$body.hits",/] -% TESTRESPONSE[s/"took": 3,/"took": "$body.took",/] +% TESTRESPONSE[s/"_shards": \.\.\./"_shards": "$body._shards",/] +% TESTRESPONSE[s/"hits": \.\.\.​/"hits": "$body.hits",/] +% TESTRESPONSE[s/"took": 3,/"took": "$body.took",/] $$$_basic_phrase_suggest_api_parameters$$$ Basic phrase suggest API parameters include: @@ -621,8 +621,8 @@ It returns this response: } } ``` -% TESTRESPONSE[s/"hits": …​/"hits": "$body.hits",/] -% TESTRESPONSE[s/"took": 2,/"took": "$body.took",/] +% TESTRESPONSE[s/"hits": \.\.\.​/"hits": "$body.hits",/] +% TESTRESPONSE[s/"took": 2,/"took": "$body.took",/] ::::{important} `_source` metadata field must be enabled, which is the default behavior, to enable returning `_source` with suggestions. @@ -1191,9 +1191,9 @@ In the response, the suggester names will be changed to respectively `term#my-fi ... } ``` -% TESTRESPONSE[s/\.\.\./"took": "$body.took", "timed_out": false, "_shards": "$body._shards", "hits": "$body.hits"/] -% TESTRESPONSE[s/"score": 0.8333333/"score": $body.suggest.term#my-first-suggester.2.options.0.score/] -% TESTRESPONSE[s/"score": 0.030227963/"score": $body.suggest.phrase#my-second-suggester.0.options.0.score/] +% TESTRESPONSE[s/\.\.\./"took": "$body.took", "timed_out": false, "_shards": "$body._shards", "hits": "$body.hits"/] +% TESTRESPONSE[s/"score": 0.8333333/"score": $body.suggest.term#my-first-suggester.2.options.0.score/] +% TESTRESPONSE[s/"score": 0.030227963/"score": $body.suggest.phrase#my-second-suggester.0.options.0.score/] 1. The name `my-first-suggester` now contains the `term` prefix. 2. The name `my-second-suggester` now contains the `phrase` prefix. diff --git a/docs/reference/enrich-processor/grok-processor.md b/docs/reference/enrich-processor/grok-processor.md index 672c0ae6c5764..2f414ae142a86 100644 --- a/docs/reference/enrich-processor/grok-processor.md +++ b/docs/reference/enrich-processor/grok-processor.md @@ -83,7 +83,7 @@ This pipeline will insert these named captures as new fields within the document ] } ``` -% TESTRESPONSE[s/2016-11-08T19:43:03.850\+0000/$body.docs.0.doc._ingest.timestamp/] +% TESTRESPONSE[s/2016-11-08T19:43:03.850\+0000/$body.docs.0.doc._ingest.timestamp/] ## Custom Patterns [custom-patterns] @@ -109,7 +109,7 @@ You can add your own patterns to a processor definition under the `pattern_defin ] } ``` -% NOTCONSOLE +% NOTCONSOLE ## Providing Multiple Match Patterns [trace-match] @@ -168,10 +168,41 @@ response: ] } ``` -% TESTRESPONSE[s/2016-11-08T19:43:03.850\+0000/$body.docs.0.doc._ingest.timestamp/] +% TESTRESPONSE[s/2016-11-08T19:43:03.850\+0000/$body.docs.0.doc._ingest.timestamp/] Both patterns will set the field `pet` with the appropriate match, but what if we want to trace which of our patterns matched and populated our fields? We can do this with the `trace_match` parameter. Here is the output of that same pipeline, but with `"trace_match": true` configured: + ```console-result { "docs": [ @@ -193,7 +224,7 @@ Both patterns will set the field `pet` with the appropriate match, but what if w ] } ``` -% TESTRESPONSE[s/2016-11-08T19:43:03.850\+0000/$body.docs.0.doc._ingest.timestamp/] +% TESTRESPONSE[s/2016-11-08T19:43:03.850\+0000/$body.docs.0.doc._ingest.timestamp/] In the above response, you can see that the index of the pattern that matched was `"1"`. This is to say that it was the second (index starts at zero) pattern in `patterns` to match. @@ -218,7 +249,7 @@ The above request will return a response body containing a key-value representat ... } ``` -% NOTCONSOLE +% NOTCONSOLE By default, the API returns a list of legacy Grok patterns. These legacy patterns predate the [Elastic Common Schema (ECS)](ecs://reference/ecs-field-reference.md) and don’t use ECS field names. To return patterns that extract ECS field names, specify `v1` in the optional `ecs_compatibility` query parameter. @@ -245,7 +276,7 @@ The API returns the following response. ... } ``` -% NOTCONSOLE +% NOTCONSOLE This can be useful to reference as the built-in patterns change across versions. @@ -276,6 +307,6 @@ PUT _cluster/settings } } ``` -% NOTCONSOLE +% NOTCONSOLE diff --git a/docs/reference/enrich-processor/normalize-for-stream.md b/docs/reference/enrich-processor/normalize-for-stream.md index 78f183c7f72ef..2c174bf9c7460 100644 --- a/docs/reference/enrich-processor/normalize-for-stream.md +++ b/docs/reference/enrich-processor/normalize-for-stream.md @@ -57,7 +57,7 @@ If the document is not OpenTelemetry-compliant, the processor normalizes it as f If an OpenTelemetry-compliant document is detected, the processor does nothing. For example, the following document will stay unchanged: -```json +```js { "resource": { "attributes": { @@ -76,10 +76,11 @@ If an OpenTelemetry-compliant document is detected, the processor does nothing. } } ``` +% NOTCONSOLE If a non-OpenTelemetry-compliant document is detected, the processor normalizes it. For example, the following document: -```json +```js { "@timestamp": "2023-10-01T12:00:00Z", "service": { @@ -118,9 +119,11 @@ If a non-OpenTelemetry-compliant document is detected, the processor normalizes "trace.id": "abcdef1234567890abcdef1234567890" } ``` +% NOTCONSOLE + will be normalized into the following form: -```json +```js { "@timestamp": "2023-10-01T12:00:00Z", "resource": { @@ -153,6 +156,8 @@ will be normalized into the following form: "trace_id": "abcdef1234567890abcdef1234567890" } ``` +% NOTCONSOLE + ## Structured `message` field If the `message` field in the ingested document is structured as a JSON, the @@ -167,16 +172,18 @@ the `body.structured` field as is, without any further normalization. For example, if the `message` field is an ECS-JSON, as follows: -```json +```js { "@timestamp": "2023-10-01T12:00:00Z", "message": "{\"@timestamp\":\"2023-10-01T12:01:00Z\",\"log.level\":\"INFO\",\"service.name\":\"my-service\",\"message\":\"The actual log message\",\"http\":{\"method\":\"GET\",\"url\":{\"path\":\"/api/v1/resource\"}}}" } ``` +% NOTCONSOLE + it will be normalized into the following form: -```json +```js { "@timestamp": "2023-10-01T12:01:00Z", "severity_text": "INFO", @@ -194,10 +201,11 @@ it will be normalized into the following form: } } ``` +% NOTCONSOLE However, if the `message` field is not recognized as ECS format, as follows: -```json +```js { "@timestamp": "2023-10-01T12:00:00Z", "log": { @@ -210,9 +218,11 @@ However, if the `message` field is not recognized as ECS format, as follows: "message": "{\"root_cause\":\"Network error\",\"http\":{\"method\":\"GET\",\"url\":{\"path\":\"/api/v1/resource\"}}}" } ``` +% NOTCONSOLE + it will be normalized into the following form: -```json +```js { "@timestamp": "2023-10-01T12:00:00Z", "severity_text": "INFO", @@ -237,3 +247,4 @@ it will be normalized into the following form: } } ``` +% NOTCONSOLE diff --git a/docs/reference/query-languages/esql/esql-cross-clusters.md b/docs/reference/query-languages/esql/esql-cross-clusters.md index 105cbd6aa2f01..f93b6f802c2e3 100644 --- a/docs/reference/query-languages/esql/esql-cross-clusters.md +++ b/docs/reference/query-languages/esql/esql-cross-clusters.md @@ -163,6 +163,9 @@ PUT _cluster/settings } } ``` +% TEST[setup:host] +% TEST[s/35.238.149.\d+:930\d+/\${transport_host}/] +% end::ccs-remote-cluster-setup[] 1. Since `skip_unavailable` was not set on `cluster_three`, it uses the default of `true`. See the [Optional remote clusters](#ccq-skip-unavailable-clusters) section for details. diff --git a/docs/reference/query-languages/query-dsl/query-dsl-distance-feature-query.md b/docs/reference/query-languages/query-dsl/query-dsl-distance-feature-query.md index d3d4c195a14e6..08240b31b37cb 100644 --- a/docs/reference/query-languages/query-dsl/query-dsl-distance-feature-query.md +++ b/docs/reference/query-languages/query-dsl/query-dsl-distance-feature-query.md @@ -79,28 +79,6 @@ To see how you can set up an index for the `distance_feature` query, try the fol The following `bool` search returns documents with a `name` value of `chocolate`. The search also uses the `distance_feature` query to increase the relevance score of documents with a `production_date` value closer to `now`. - - ```console GET /items/_search { diff --git a/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md b/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md index 538db88600fa2..b31df82e5704f 100644 --- a/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md +++ b/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md @@ -223,7 +223,7 @@ To ensure correct results: each individual filter must be either over: This query performs a basic nested knn search: -```json +```js { "query" : { "nested" : { @@ -238,6 +238,7 @@ This query performs a basic nested knn search: } } ``` +% NOTCONSOLE ### Filter over nested metadata @@ -248,7 +249,7 @@ stack: ga 9.2 This query filters over nested metadata. For scoring parent documents, this query only considers vectors that have "paragraph.language" set to "EN": -```json +```js { "query" : { "nested" : { @@ -268,6 +269,7 @@ have "paragraph.language" set to "EN": } } ``` +% NOTCONSOLE ### Multiple filters (nested and top-level metadata) @@ -279,7 +281,7 @@ This query uses multiple filters: one over nested metadata and another over the this query only considers vectors whose parent's title contain "essay" word and have "paragraph.language" set to "EN": -```json +```js { "query" : { "nested" : { @@ -306,6 +308,7 @@ word and have "paragraph.language" set to "EN": } } ``` +% NOTCONSOLE Note that nested `knn` only supports `score_mode=max`. @@ -316,7 +319,7 @@ Elasticsearch supports knn queries over a [ Here is an example using the `query_vector_builder`: -```json +```js { "query": { "knn": { @@ -332,6 +335,7 @@ Here is an example using the `query_vector_builder`: } } ``` +% NOTCONSOLE Note that for `semantic_text` fields, the `model_id` does not have to be provided as it can be inferred from the `semantic_text` field mapping. diff --git a/docs/reference/query-languages/query-dsl/query-dsl-parent-id-query.md b/docs/reference/query-languages/query-dsl/query-dsl-parent-id-query.md index 0df12d96fd120..2858a52e9fb0c 100644 --- a/docs/reference/query-languages/query-dsl/query-dsl-parent-id-query.md +++ b/docs/reference/query-languages/query-dsl/query-dsl-parent-id-query.md @@ -32,7 +32,7 @@ To use the `parent_id` query, your index must include a [join](/reference/elasti } } ``` - % TESTSETUP + % TESTSETUP 2. Index a parent document with an ID of `1`. @@ -63,24 +63,6 @@ To use the `parent_id` query, your index must include a [join](/reference/elasti The following search returns child documents for a parent document with an ID of `1`. - ```console GET /my-index-000001/_search { diff --git a/docs/reference/query-languages/query-dsl/query-dsl-script-score-query.md b/docs/reference/query-languages/query-dsl/query-dsl-script-score-query.md index cd5940e6255ca..b6ba37d3ba5e0 100644 --- a/docs/reference/query-languages/query-dsl/query-dsl-script-score-query.md +++ b/docs/reference/query-languages/query-dsl/query-dsl-script-score-query.md @@ -30,7 +30,6 @@ GET /_search } } ``` -% TEARDOWN ## Top-level parameters for `script_score` [script-score-top-level-params] @@ -370,7 +369,6 @@ PUT my-index-000001/_doc/2 POST my-index-000001/_refresh ``` -% TESTSETUP #### Cosine similarity [vector-functions-cosine] @@ -400,6 +398,7 @@ GET my-index-000001/_search } } ``` +% TEST[continued] 1. To restrict the number of documents on which script score calculation is applied, provide a filter. 2. The script adds 1.0 to the cosine similarity to prevent the score from being negative. @@ -443,6 +442,7 @@ GET my-index-000001/_search } } ``` +% TEST[continued] 1. Using the standard sigmoid function prevents scores from being negative. @@ -476,6 +476,7 @@ GET my-index-000001/_search } } ``` +% TEST[continued] 1. Unlike `cosineSimilarity` that represent similarity, `l1norm` and `l2norm` shown below represent distances or differences. This means, that the more similar the vectors are, the lower the scores will be that are produced by the `l1norm` and `l2norm` functions. Thus, as we need more similar vectors to score higher, we reversed the output from `l1norm` and `l2norm`. Also, to avoid division by 0 when a document vector matches the query exactly, we added `1` in the denominator. @@ -509,6 +510,7 @@ GET my-index-000001/_search } } ``` +% TEST[continued] 1. Calculate the Hamming distance and normalize it by the bits to get a score between 0 and 1. @@ -542,7 +544,7 @@ GET my-index-000001/_search } } ``` - +% TEST[continued] #### Checking for missing values [vector-functions-missing-values] @@ -741,7 +743,7 @@ GET /my-index-000001/_explain/0 } } ``` -% TEST[setup:my_index] +% TEST[continued] Note that the `explanation` will be null when using in a normal `_search` request, so having a conditional guard is best practice. diff --git a/docs/reference/query-languages/query-dsl/query-dsl-terms-set-query.md b/docs/reference/query-languages/query-dsl/query-dsl-terms-set-query.md index 4a3cb367a881b..6d4e913f78e3a 100644 --- a/docs/reference/query-languages/query-dsl/query-dsl-terms-set-query.md +++ b/docs/reference/query-languages/query-dsl/query-dsl-terms-set-query.md @@ -95,28 +95,6 @@ The following search returns documents where the `programming_languages` field c The `minimum_should_match_field` is `required_matches`. This means the number of matching terms required is `2`, the value of the `required_matches` field. - - ```console GET /job-candidates/_search { diff --git a/docs/reference/scripting-languages/painless/painless-watcher-condition-context.md b/docs/reference/scripting-languages/painless/painless-watcher-condition-context.md index 9d257994c0f20..9cbf798c7d175 100644 --- a/docs/reference/scripting-languages/painless/painless-watcher-condition-context.md +++ b/docs/reference/scripting-languages/painless/painless-watcher-condition-context.md @@ -96,6 +96,7 @@ POST _watcher/watch/_execute } } ``` +% TEST[setup:seats] 1. The Java Stream API is used in the condition. This API allows manipulation of the elements of the list in a pipeline. 2. The stream filter removes items that do not meet the filter criteria. @@ -154,6 +155,7 @@ POST _watcher/watch/_execute } } ``` +% TEST[setup:seats] This example uses a nearly identical condition as the previous example. The differences below are subtle and are worth calling out. diff --git a/docs/reference/scripting-languages/painless/painless-watcher-transform-context.md b/docs/reference/scripting-languages/painless/painless-watcher-transform-context.md index 3c68fc7f68d0e..5b7afcabf10ce 100644 --- a/docs/reference/scripting-languages/painless/painless-watcher-transform-context.md +++ b/docs/reference/scripting-languages/painless/painless-watcher-transform-context.md @@ -106,6 +106,7 @@ POST _watcher/watch/_execute } } ``` +% TEST[setup:seats] 1. The Java Stream API is used in the transform. This API allows manipulation of the elements of the list in a pipeline. 2. The stream filter removes items that do not meet the filter criteria. @@ -169,6 +170,7 @@ POST _watcher/watch/_execute } } ``` +% TEST[setup:seats] This example uses the streaming API in a very similar manner. The differences below are subtle and worth calling out. @@ -283,6 +285,7 @@ POST _watcher/watch/_execute } } ``` +% TEST[setup:seats] The following example shows the use of metadata and transforming dates into a readable format. @@ -338,4 +341,4 @@ POST _watcher/watch/_execute } } ``` - +% TEST[setup:seats] diff --git a/docs/reference/search-connectors/es-connectors-dropbox.md b/docs/reference/search-connectors/es-connectors-dropbox.md index 7af5b16080f9c..a4070e80c2f12 100644 --- a/docs/reference/search-connectors/es-connectors-dropbox.md +++ b/docs/reference/search-connectors/es-connectors-dropbox.md @@ -51,7 +51,7 @@ PUT _connector/my-dropbox-connector "service_type": "dropbox" } ``` -% TEST[skip:can’t test in isolation] +% TEST[skip:can’t test in isolation] :::::{dropdown} You’ll also need to create an API key for the connector to use. ::::{note} @@ -145,6 +145,7 @@ To generate a refresh token, follow these steps: ```shell curl -X POST "https://api.dropboxapi.com/oauth2/token?code=&grant_type=authorization_code" -u ":" ``` + % NOTCONSOLE Store the refresh token from the response to be used in the connector configuration. @@ -215,7 +216,7 @@ Download the sample configuration file. You can either download it manually or r ```sh curl https://raw.githubusercontent.com/elastic/connectors/main/config.yml.example --output ~/connectors-config/config.yml ``` -% NOTCONSOLE +% NOTCONSOLE Remember to update the `--output` argument value if your directory name is different, or you want to use a different config file name. @@ -242,6 +243,7 @@ connectors: service_type: dropbox api_key: # Optional. If not provided, the connector will use the elasticsearch.api_key instead ``` +% NOTCONSOLE Using the `elasticsearch.api_key` is the recommended authentication method. However, you can also use `elasticsearch.username` and `elasticsearch.password` to authenticate with your Elasticsearch instance. @@ -263,7 +265,7 @@ docker.elastic.co/integrations/elastic-connectors:{{version.stack}} \ /app/bin/elastic-ingest \ -c /config/config.yml ``` - +% NOTCONSOLE :::: @@ -340,7 +342,7 @@ $$$es-connectors-dropbox-client-sync-rules-advanced-example-1$$$ } ] ``` -% NOTCONSOLE +% NOTCONSOLE $$$es-connectors-dropbox-client-sync-rules-advanced-example-2$$$ **Example: Query with file extension filter** @@ -358,7 +360,7 @@ $$$es-connectors-dropbox-client-sync-rules-advanced-example-2$$$ } ] ``` -% NOTCONSOLE +% NOTCONSOLE $$$es-connectors-dropbox-client-sync-rules-advanced-example-3$$$ **Example: Query with file category filter** @@ -380,7 +382,7 @@ $$$es-connectors-dropbox-client-sync-rules-advanced-example-3$$$ } ] ``` -% NOTCONSOLE +% NOTCONSOLE $$$es-connectors-dropbox-client-sync-rules-advanced-limitations$$$ **Limitations** @@ -404,7 +406,6 @@ For faster tests, add the `DATA_SIZE=small` flag: make ftest NAME=dropbox DATA_SIZE=small ``` - ### Known issues [es-connectors-dropbox-client-known-issues] Refer to [Known issues](/release-notes/known-issues.md) for a list of known issues for all connectors. diff --git a/docs/reference/search-connectors/es-connectors-graphql.md b/docs/reference/search-connectors/es-connectors-graphql.md index 7c9059820baa6..84a3d23297f75 100644 --- a/docs/reference/search-connectors/es-connectors-graphql.md +++ b/docs/reference/search-connectors/es-connectors-graphql.md @@ -45,7 +45,7 @@ Download the sample configuration file. You can either download it manually or r ```sh curl https://raw.githubusercontent.com/elastic/connectors/main/config.yml.example --output ~/connectors-config/config.yml ``` -% NOTCONSOLE +% NOTCONSOLE Remember to update the `--output` argument value if your directory name is different, or you want to use a different config file name. diff --git a/docs/reference/search-connectors/es-connectors-known-issues.md b/docs/reference/search-connectors/es-connectors-known-issues.md index 9d7063911f681..9b043e6acf734 100644 --- a/docs/reference/search-connectors/es-connectors-known-issues.md +++ b/docs/reference/search-connectors/es-connectors-known-issues.md @@ -58,6 +58,7 @@ The connector service has the following known issues: } } ``` + % TEST[skip:TODO] * **The connector service will fail to sync when the connector tries to fetch more more than 2,147,483,647 (*2^31-1*) documents from a data source** @@ -75,6 +76,7 @@ The connector service has the following known issues: } } ``` + % TEST[skip:TODO] This error can appear on Connectors or Crawlers that aren’t the cause of the issue. If the error continues, try running the above command for every document in the `.elastic-connectors` index. @@ -112,6 +114,7 @@ The connector service has the following known issues: } } ``` + % TEST[skip:TODO] * **Python connectors that upgraded from 8.7.1 will report document volumes in gigabytes (GB) instead of megabytes (MB)** diff --git a/docs/reference/search-connectors/es-connectors-mysql.md b/docs/reference/search-connectors/es-connectors-mysql.md index d96f2394c6570..ebc8e7f21985c 100644 --- a/docs/reference/search-connectors/es-connectors-mysql.md +++ b/docs/reference/search-connectors/es-connectors-mysql.md @@ -48,7 +48,7 @@ PUT _connector/my-mysql-connector "service_type": "mysql" } ``` -% TEST[skip:can’t test in isolation] +% TEST[skip:can’t test in isolation] :::::{dropdown} You’ll also need to create an API key for the connector to use. ::::{note} @@ -233,6 +233,7 @@ This connector has the following known issues: } } ``` + % TEST[skip:TODO] * **Upgrading to 8.8 does not migrate MySQL sync rules.** diff --git a/docs/reference/search-connectors/es-dls-e2e-guide.md b/docs/reference/search-connectors/es-dls-e2e-guide.md index 77c2f5b558e4b..c0339eeec1571 100644 --- a/docs/reference/search-connectors/es-dls-e2e-guide.md +++ b/docs/reference/search-connectors/es-dls-e2e-guide.md @@ -116,7 +116,7 @@ The access control index will contain documents similar to this example: } } ``` -% NOTCONSOLE +% NOTCONSOLE This document contains the Elasticsearch query that describes which documents the user `john@example.com` has access to. The access control information is stored in the `access_control` field. In this case the user has access only to documents that contain `"john@example.co"` or `"Engineering Members"` in the `_allow_access_control` field. @@ -182,7 +182,7 @@ POST /_security/api_key } } ``` -% TEST[skip:TODO] +% TEST[skip:TODO] The response will look like this: @@ -195,7 +195,7 @@ The response will look like this: "encoded": "Qk05dy1JZ0JhRDNyNGpLQ3MwUmk6elRzdGU5QjZUY21SSWdkMldnQ1RMZw==" } ``` -% NOTCONSOLE +% NOTCONSOLE The `api_key` field contains the API key that can be used to query the Search Application with the appropriate DLS restrictions. @@ -228,7 +228,7 @@ GET .search-acl-filter-source1 } } ``` -% NOTCONSOLE +% NOTCONSOLE ```js GET .search-acl-filter-source2 @@ -250,7 +250,7 @@ GET .search-acl-filter-source2 } } ``` -% NOTCONSOLE +% NOTCONSOLE `.search-acl-filter-source1` and `.search-acl-filter-source2` define the access control identities for `source1` and `source2`. @@ -357,7 +357,7 @@ createApiKey({ }, }).then((encodedKey) => console.log(encodedKey)); ``` -% NOTCONSOLE +% NOTCONSOLE ::::{note} The example combines multiple identities into a single role descriptor. This is because an Elasticsearch API key can use role restrictions only if it has a **single role descriptor**. @@ -377,6 +377,7 @@ If you’re building a frontend application, use the `encoded` field to pass the ```js const client = SearchApplicationClient(applicationName, endpoint, apiKey, params); ``` + % NOTCONSOLE Here’s what this workflow looks like in a sequence diagram: