From 9f52f46ae341ae0ff46522c6eb2b996fee8d7ac2 Mon Sep 17 00:00:00 2001 From: Colleen McGinnis Date: Thu, 6 Mar 2025 14:00:09 -0600 Subject: [PATCH 1/4] add es code comments for testing --- ...asticsearch-from-archive-on-linux-macos.md | 2 + ...stall-elasticsearch-with-debian-package.md | 1 + .../install-elasticsearch-with-rpm.md | 1 + ...stall-elasticsearch-with-zip-on-windows.md | 1 + ...cal-development-installation-quickstart.md | 1 + .../delaying-allocation-when-node-leaves.md | 2 + .../add-and-remove-elasticsearch-nodes.md | 1 + .../elasticsearch-deprecation-logs.md | 1 + ...search-log4j-configuration-self-managed.md | 1 + .../update-elasticsearch-logging-levels.md | 1 + .../optimize-performance/size-shards.md | 21 ++++++ .../remote-clusters-api-key.md | 17 +++++ .../remote-clusters/remote-clusters-cert.md | 15 ++++ .../snapshot-and-restore/azure-repository.md | 3 + .../snapshot-and-restore/create-snapshots.md | 7 ++ .../google-cloud-storage-repository.md | 4 ++ .../read-only-url-repository.md | 1 + .../snapshot-and-restore/restore-snapshot.md | 18 +++++ .../snapshot-and-restore/s3-repository.md | 7 ++ .../snapshot-and-restore/self-managed.md | 7 ++ .../shared-file-system-repository.md | 6 ++ .../source-only-repository.md | 2 + .../archived-settings.md | 2 + .../cluster-or-deployment-auth/built-in-sm.md | 1 + ...ing-privileges-for-data-streams-aliases.md | 8 +++ .../role-mapping-resources.md | 1 + .../service-accounts.md | 2 + .../token-based-authentication-services.md | 1 + .../alerts-cases/watcher/actions-email.md | 2 + .../alerts-cases/watcher/actions-index.md | 2 + .../alerts-cases/watcher/actions-jira.md | 1 + .../alerts-cases/watcher/actions-logging.md | 1 + .../alerts-cases/watcher/actions-pagerduty.md | 2 + .../alerts-cases/watcher/actions-slack.md | 3 + .../alerts-cases/watcher/actions-webhook.md | 4 ++ .../alerts-cases/watcher/actions.md | 1 + .../alerts-cases/watcher/condition-always.md | 1 + .../watcher/condition-array-compare.md | 1 + .../alerts-cases/watcher/condition-compare.md | 3 + .../alerts-cases/watcher/condition-never.md | 1 + .../alerts-cases/watcher/condition-script.md | 7 ++ .../alerts-cases/watcher/how-watcher-works.md | 8 +++ .../alerts-cases/watcher/input-chain.md | 1 + .../alerts-cases/watcher/input-http.md | 7 ++ .../alerts-cases/watcher/input-search.md | 5 ++ .../alerts-cases/watcher/input-simple.md | 2 + .../alerts-cases/watcher/schedule-types.md | 23 ++++++ .../alerts-cases/watcher/transform-chain.md | 1 + .../alerts-cases/watcher/transform-script.md | 1 + .../alerts-cases/watcher/transform-search.md | 4 ++ .../alerts-cases/watcher/transform.md | 1 + .../watcher/watch-cluster-status.md | 5 ++ .../watcher/watcher-getting-started.md | 4 ++ ...bacloud-ai-search-inference-integration.md | 4 ++ .../amazon-bedrock-inference-integration.md | 2 + .../anthropic-inference-integration.md | 1 + .../azure-ai-studio-inference-integration.md | 2 + .../azure-openai-inference-integration.md | 2 + .../chat-completion-inference-api.md | 4 ++ .../cohere-inference-integration.md | 2 + .../elastic-inference-service-eis.md | 2 + .../elasticsearch-inference-integration.md | 7 ++ .../elser-inference-integration.md | 3 + .../google-ai-studio-inference-integration.md | 1 + .../google-vertex-ai-inference-integration.md | 2 + .../huggingface-inference-integration.md | 1 + .../jinaai-inference-integration.md | 6 ++ .../mistral-inference-integration.md | 1 + .../openai-inference-integration.md | 2 + .../nlp/nlp-end-to-end-tutorial.md | 5 ++ explore-analyze/query-filter/aggregations.md | 29 ++++++++ ...-data-with-aggregations-using-query-dsl.md | 16 +++++ explore-analyze/query-filter/languages/eql.md | 72 +++++++++++++++++++ .../languages/esql-cross-clusters.md | 7 ++ .../query-filter/languages/esql-rest.md | 20 ++++++ .../languages/esql-task-management.md | 1 + .../example-detect-threats-with-eql.md | 16 +++++ .../query-filter/languages/sql-async.md | 23 ++++++ .../query-filter/languages/sql-data-types.md | 1 + .../languages/sql-getting-started.md | 3 + .../query-filter/languages/sql-pagination.md | 5 ++ .../languages/sql-rest-columnar.md | 5 ++ .../languages/sql-rest-filtering.md | 4 ++ .../query-filter/languages/sql-rest-format.md | 14 ++++ .../languages/sql-rest-overview.md | 3 + .../query-filter/languages/sql-rest-params.md | 2 + .../languages/sql-runtime-fields.md | 2 + .../query-filter/languages/sql-translate.md | 1 + explore-analyze/scripting/dissect.md | 11 +++ explore-analyze/scripting/grok.md | 7 ++ .../scripting/modules-scripting-engine.md | 1 + .../scripting/modules-scripting-using.md | 15 ++++ .../scripting/scripting-field-extraction.md | 14 ++++ .../scripting/scripts-search-speed.md | 6 ++ .../transforms/transform-api-quickref.md | 1 + .../transforms/transform-examples.md | 16 +++++ .../transforms/transform-painless-examples.md | 9 +++ explore-analyze/transforms/transform-scale.md | 1 + manage-data/data-store/aliases.md | 16 +++++ .../data-streams/logs-data-stream.md | 1 + .../data-streams/modify-data-stream.md | 1 + .../data-store/data-streams/reindex-tsds.md | 6 ++ .../data-streams/run-downsampling-manually.md | 17 +++++ ...ownsampling-using-data-stream-lifecycle.md | 17 +++++ .../data-streams/run-downsampling-with-ilm.md | 17 +++++ .../data-streams/set-up-data-stream.md | 8 +++ .../data-store/data-streams/set-up-tsds.md | 4 ++ .../data-streams/use-data-stream.md | 17 +++++ ...define-runtime-fields-in-search-request.md | 6 ++ .../data-store/mapping/dynamic-templates.md | 6 ++ .../data-store/mapping/explicit-mapping.md | 3 + .../explore-data-with-runtime-fields.md | 18 +++++ .../data-store/mapping/index-runtime-field.md | 9 +++ .../data-store/mapping/map-runtime-field.md | 1 + .../override-field-values-at-query-time.md | 8 +++ .../mapping/retrieve-runtime-field.md | 8 +++ .../text-analysis/specify-an-analyzer.md | 1 + ...ample-enrich-data-based-on-exact-values.md | 7 ++ ...xample-enrich-data-based-on-geolocation.md | 8 +++ ...-enrich-data-by-matching-value-to-range.md | 8 +++ .../transform-enrich/example-parse-logs.md | 1 + .../transform-enrich/ingest-pipelines.md | 10 +++ ...orial-create-data-stream-with-lifecycle.md | 3 + .../tutorial-data-stream-retention.md | 2 + ...ed-data-stream-to-data-stream-lifecycle.md | 35 +++++++++ .../tutorial-update-existing-data-stream.md | 7 ++ .../configure-lifecycle-policy.md | 1 + ...-index-allocation-filters-to-node-roles.md | 4 ++ .../start-stop-index-lifecycle-management.md | 4 ++ .../tutorial-automate-rollover.md | 7 ++ .../lifecycle/rollup/getting-started-api.md | 14 ++++ .../migrating-from-rollup-to-downsampling.md | 1 + .../rollup/rollup-search-limitations.md | 4 ++ .../lifecycle/rollup/understanding-groups.md | 7 ++ ...lasticsearch-to-manage-time-series-data.md | 7 ++ solutions/search/cross-cluster-search.md | 62 ++++++++++++++++ .../mixing-exact-search-with-stemming.md | 6 ++ .../static-scoring-signals.md | 1 + solutions/search/hybrid-semantic-text.md | 3 + .../querydsl-full-text-filter-tutorial.md | 24 +++++++ .../search/ranking/learning-to-rank-ltr.md | 1 + .../learning-to-rank-model-training.md | 4 ++ .../ranking/learning-to-rank-search-usage.md | 1 + .../search/ranking/semantic-reranking.md | 1 + solutions/search/retrievers-overview.md | 2 + solutions/search/run-elasticsearch-locally.md | 1 + solutions/search/search-applications.md | 1 + .../search-application-api.md | 20 ++++++ .../search-application-client.md | 9 +++ .../search-application-security.md | 4 ++ solutions/search/search-templates.md | 14 ++++ solutions/search/semantic-search/cohere-es.md | 1 + .../semantic-search-elser-ingest-pipelines.md | 8 +++ .../semantic-search-inference.md | 52 ++++++++++++++ .../semantic-search-semantic-text.md | 5 ++ solutions/search/vector/bring-own-vectors.md | 4 ++ .../dense-versus-sparse-ingest-pipelines.md | 8 +++ troubleshoot/elasticsearch/add-tier.md | 2 + .../elasticsearch/circuit-breaker-errors.md | 2 + .../elasticsearch/fix-watermark-errors.md | 3 + troubleshoot/elasticsearch/high-cpu-usage.md | 2 + .../elasticsearch/high-jvm-memory-pressure.md | 1 + troubleshoot/elasticsearch/hotspotting.md | 4 ++ .../increase-cluster-shard-limit.md | 2 + .../elasticsearch/increase-shard-limit.md | 2 + .../elasticsearch/increase-tier-capacity.md | 4 ++ .../index-lifecycle-management-errors.md | 6 ++ .../red-yellow-cluster-status.md | 3 + troubleshoot/elasticsearch/remote-clusters.md | 6 ++ .../repeated-snapshot-failures.md | 3 + .../elasticsearch/troubleshooting-searches.md | 11 +++ .../troubleshooting-shards-capacity-issues.md | 5 ++ .../troubleshooting-unbalanced-cluster.md | 3 + .../elasticsearch/watcher-troubleshooting.md | 1 + 174 files changed, 1164 insertions(+) diff --git a/deploy-manage/deploy/self-managed/install-elasticsearch-from-archive-on-linux-macos.md b/deploy-manage/deploy/self-managed/install-elasticsearch-from-archive-on-linux-macos.md index bd23611b13..c340fa9aad 100644 --- a/deploy-manage/deploy/self-managed/install-elasticsearch-from-archive-on-linux-macos.md +++ b/deploy-manage/deploy/self-managed/install-elasticsearch-from-archive-on-linux-macos.md @@ -69,6 +69,7 @@ curl https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-9.0.0-be tar -xzf elasticsearch-9.0.0-beta1-darwin-x86_64.tar.gz cd elasticsearch-9.0.0-beta1/ <2> ``` +% NOTCONSOLE 1. Compares the SHA of the downloaded `.tar.gz` archive and the published checksum, which should output `elasticsearch-{{version}}-darwin-x86_64.tar.gz: OK`. 2. This directory is known as `$ES_HOME`. @@ -164,6 +165,7 @@ You can test that your {{es}} node is running by sending an HTTPS request to por ```sh curl --cacert $ES_HOME/config/certs/http_ca.crt -u elastic:$ELASTIC_PASSWORD https://localhost:9200 <1> ``` +% NOTCONSOLE 1. Ensure that you use `https` in your call, or the request will fail.`--cacert` : Path to the generated `http_ca.crt` certificate for the HTTP layer. diff --git a/deploy-manage/deploy/self-managed/install-elasticsearch-with-debian-package.md b/deploy-manage/deploy/self-managed/install-elasticsearch-with-debian-package.md index ba6e48d296..58d6834d36 100644 --- a/deploy-manage/deploy/self-managed/install-elasticsearch-with-debian-package.md +++ b/deploy-manage/deploy/self-managed/install-elasticsearch-with-debian-package.md @@ -211,6 +211,7 @@ You can test that your {{es}} node is running by sending an HTTPS request to por ```sh curl --cacert /etc/elasticsearch/certs/http_ca.crt -u elastic:$ELASTIC_PASSWORD https://localhost:9200 <1> ``` +% NOTCONSOLE 1. Ensure that you use `https` in your call, or the request will fail.`--cacert` : Path to the generated `http_ca.crt` certificate for the HTTP layer. diff --git a/deploy-manage/deploy/self-managed/install-elasticsearch-with-rpm.md b/deploy-manage/deploy/self-managed/install-elasticsearch-with-rpm.md index be47ceb5c2..a3288c8bf6 100644 --- a/deploy-manage/deploy/self-managed/install-elasticsearch-with-rpm.md +++ b/deploy-manage/deploy/self-managed/install-elasticsearch-with-rpm.md @@ -215,6 +215,7 @@ You can test that your {{es}} node is running by sending an HTTPS request to por ```sh curl --cacert /etc/elasticsearch/certs/http_ca.crt -u elastic:$ELASTIC_PASSWORD https://localhost:9200 <1> ``` +% NOTCONSOLE 1. Ensure that you use `https` in your call, or the request will fail.`--cacert` : Path to the generated `http_ca.crt` certificate for the HTTP layer. diff --git a/deploy-manage/deploy/self-managed/install-elasticsearch-with-zip-on-windows.md b/deploy-manage/deploy/self-managed/install-elasticsearch-with-zip-on-windows.md index ca5b5a3635..b905ff7c78 100644 --- a/deploy-manage/deploy/self-managed/install-elasticsearch-with-zip-on-windows.md +++ b/deploy-manage/deploy/self-managed/install-elasticsearch-with-zip-on-windows.md @@ -142,6 +142,7 @@ You can test that your {{es}} node is running by sending an HTTPS request to por ```sh curl --cacert %ES_HOME%\config\certs\http_ca.crt -u elastic:$ELASTIC_PASSWORD https://localhost:9200 <1> ``` +% NOTCONSOLE 1. Ensure that you use `https` in your call, or the request will fail.`--cacert` : Path to the generated `http_ca.crt` certificate for the HTTP layer. diff --git a/deploy-manage/deploy/self-managed/local-development-installation-quickstart.md b/deploy-manage/deploy/self-managed/local-development-installation-quickstart.md index 48ede71a67..27be920769 100644 --- a/deploy-manage/deploy/self-managed/local-development-installation-quickstart.md +++ b/deploy-manage/deploy/self-managed/local-development-installation-quickstart.md @@ -35,6 +35,7 @@ To set up {{es}} and {{kib}} locally, run the `start-local` script: ```sh curl -fsSL https://elastic.co/start-local | sh ``` +% NOTCONSOLE This script creates an `elastic-start-local` folder containing configuration files and starts both {{es}} and {{kib}} using Docker. diff --git a/deploy-manage/distributed-architecture/shard-allocation-relocation-recovery/delaying-allocation-when-node-leaves.md b/deploy-manage/distributed-architecture/shard-allocation-relocation-recovery/delaying-allocation-when-node-leaves.md index 3bffc7e50d..48b9aabad7 100644 --- a/deploy-manage/distributed-architecture/shard-allocation-relocation-recovery/delaying-allocation-when-node-leaves.md +++ b/deploy-manage/distributed-architecture/shard-allocation-relocation-recovery/delaying-allocation-when-node-leaves.md @@ -37,6 +37,7 @@ PUT _all/_settings } } ``` +% TEST[s/^/PUT test\n/] With delayed allocation enabled, the above scenario changes to look like this: @@ -83,6 +84,7 @@ PUT _all/_settings } } ``` +% TEST[s/^/PUT test\n/] You can reset the timeout as soon as the missing shards have started to recover. diff --git a/deploy-manage/maintenance/add-and-remove-elasticsearch-nodes.md b/deploy-manage/maintenance/add-and-remove-elasticsearch-nodes.md index 175a087eae..e48d71540b 100644 --- a/deploy-manage/maintenance/add-and-remove-elasticsearch-nodes.md +++ b/deploy-manage/maintenance/add-and-remove-elasticsearch-nodes.md @@ -107,6 +107,7 @@ POST /_cluster/voting_config_exclusions?node_names=node_name # auto-reconfiguration up to one minute POST /_cluster/voting_config_exclusions?node_names=node_name&timeout=1m ``` +% TEST[skip:this would break the test cluster if executed] The nodes that should be added to the exclusions list are specified by name using the `?node_names` query parameter, or by their persistent node IDs using the `?node_ids` query parameter. If a call to the voting configuration exclusions API fails, you can safely retry it. Only a successful response guarantees that the node has actually been removed from the voting configuration and will not be reinstated. If the elected master node is excluded from the voting configuration then it will abdicate to another master-eligible node that is still in the voting configuration if such a node is available. diff --git a/deploy-manage/monitor/logging-configuration/elasticsearch-deprecation-logs.md b/deploy-manage/monitor/logging-configuration/elasticsearch-deprecation-logs.md index d1d86b254e..a3bfd243d5 100644 --- a/deploy-manage/monitor/logging-configuration/elasticsearch-deprecation-logs.md +++ b/deploy-manage/monitor/logging-configuration/elasticsearch-deprecation-logs.md @@ -259,6 +259,7 @@ You can identify what is triggering deprecated functionality if `X-Opaque-Id` wa "node.id": "D7fUYfnfTLa2D7y-xw6tZg" } ``` +% NOTCONSOLE Deprecation logs can be indexed into `.logs-deprecation.elasticsearch-default` data stream `cluster.deprecation_indexing.enabled` setting is set to true. diff --git a/deploy-manage/monitor/logging-configuration/elasticsearch-log4j-configuration-self-managed.md b/deploy-manage/monitor/logging-configuration/elasticsearch-log4j-configuration-self-managed.md index 4277d3ca26..efa886f2ff 100644 --- a/deploy-manage/monitor/logging-configuration/elasticsearch-log4j-configuration-self-managed.md +++ b/deploy-manage/monitor/logging-configuration/elasticsearch-log4j-configuration-self-managed.md @@ -256,6 +256,7 @@ You can identify what is triggering deprecated functionality if `X-Opaque-Id` wa "node.id": "D7fUYfnfTLa2D7y-xw6tZg" } ``` +% NOTCONSOLE Deprecation logs can be indexed into `.logs-deprecation.elasticsearch-default` data stream `cluster.deprecation_indexing.enabled` setting is set to true. diff --git a/deploy-manage/monitor/logging-configuration/update-elasticsearch-logging-levels.md b/deploy-manage/monitor/logging-configuration/update-elasticsearch-logging-levels.md index 63bd147452..e5768c82ee 100644 --- a/deploy-manage/monitor/logging-configuration/update-elasticsearch-logging-levels.md +++ b/deploy-manage/monitor/logging-configuration/update-elasticsearch-logging-levels.md @@ -259,6 +259,7 @@ You can identify what is triggering deprecated functionality if `X-Opaque-Id` wa "node.id": "D7fUYfnfTLa2D7y-xw6tZg" } ``` +% NOTCONSOLE Deprecation logs can be indexed into `.logs-deprecation.elasticsearch-default` data stream `cluster.deprecation_indexing.enabled` setting is set to true. diff --git a/deploy-manage/production-guidance/optimize-performance/size-shards.md b/deploy-manage/production-guidance/optimize-performance/size-shards.md index aa7b654ec9..43af6790f8 100644 --- a/deploy-manage/production-guidance/optimize-performance/size-shards.md +++ b/deploy-manage/production-guidance/optimize-performance/size-shards.md @@ -89,6 +89,7 @@ To see the current size of your shards, use the [cat shards API](https://www.ela ```console GET _cat/shards?v=true&h=index,prirep,shard,store&s=prirep,store&bytes=gb ``` +% TEST[setup:my_index] The `pri.store.size` value shows the combined size of all primary shards for the index. @@ -97,6 +98,9 @@ index prirep shard store .ds-my-data-stream-2099.05.06-000001 p 0 50gb ... ``` +% TESTRESPONSE[non_json] +% TESTRESPONSE[s/.ds-my-data-stream-2099.05.06-000001/my-index-000001/] +% TESTRESPONSE[s/50gb/.*/] If an index’s shard is experiencing degraded performance from surpassing the recommended 50GB size, you may consider fixing the index’s shards' sizing. Shards are immutable and therefore their size is fixed in place, so indices must be copied with corrected settings. This requires first ensuring sufficient disk to copy the data. Afterwards, you can copy the index’s data with corrected settings via one of the following options: @@ -123,12 +127,14 @@ To check the configured size of each node’s heap, use the [cat nodes API](http ```console GET _cat/nodes?v=true&h=heap.max ``` +% TEST[setup:my_index] You can use the [cat shards API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-cat-shards) to check the number of shards per node. ```console GET _cat/shards?v=true ``` +% TEST[setup:my_index] ### Add enough nodes to stay within the cluster shard limits [shard-count-per-node-recommendation] @@ -148,6 +154,7 @@ Each node in the cluster has a copy of the [cluster state](https://www.elastic.c ```console GET _cluster/stats?human&filter_path=indices.mappings.total_deduplicated_mapping_size* ``` +% TEST[setup:node] This will show you information like in this example output: @@ -161,6 +168,8 @@ This will show you information like in this example output: } } ``` +% TESTRESPONSE[s/"total_deduplicated_mapping_size": "1gb"/"total_deduplicated_mapping_size": $body.$_path/] +% TESTRESPONSE[s/"total_deduplicated_mapping_size_in_bytes": 1073741824/"total_deduplicated_mapping_size_in_bytes": $body.$_path/] #### Retrieving heap size and field mapper overheads [_retrieving_heap_size_and_field_mapper_overheads] @@ -173,6 +182,7 @@ You can use the [Nodes stats API](https://www.elastic.co/docs/api/doc/elasticsea ```console GET _nodes/stats?human&filter_path=nodes.*.name,nodes.*.indices.mappings.total_estimated_overhead*,nodes.*.jvm.mem.heap_max* ``` +% TEST[setup:node] For each node, this will show you information like in this example output: @@ -197,6 +207,12 @@ For each node, this will show you information like in this example output: } } ``` +% TESTRESPONSE[s/"USpTGYaBSIKbgSUJR2Z9lg"/$node_name/] +% TESTRESPONSE[s/"name": "node-0"/"name": $body.$_path/] +% TESTRESPONSE[s/"total_estimated_overhead": "1gb"/"total_estimated_overhead": $body.$_path/] +% TESTRESPONSE[s/"total_estimated_overhead_in_bytes": 1073741824/"total_estimated_overhead_in_bytes": $body.$_path/] +% TESTRESPONSE[s/"heap_max": "4gb"/"heap_max": $body.$_path/] +% TESTRESPONSE[s/"heap_max_in_bytes": 4294967296/"heap_max_in_bytes": $body.$_path/] #### Consider additional heap overheads [_consider_additional_heap_overheads] @@ -233,6 +249,7 @@ PUT my-index-000001/_settings } } ``` +% TEST[setup:my_index] ### Avoid unnecessary mapped fields [avoid-unnecessary-fields] @@ -263,12 +280,14 @@ You can find these empty indices using the [cat count API](https://www.elastic.c ```console GET _cat/count/my-index-000001?v=true ``` +% TEST[setup:my_index] Once you have a list of empty indices, you can delete them using the [delete index API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-delete). You can also delete any other unneeded indices. ```console DELETE my-index-000001 ``` +% TEST[setup:my_index] ### Force merge during off-peak hours [force-merge-during-off-peak-hours] @@ -278,6 +297,7 @@ If you no longer write to an index, you can use the [force merge API](https://ww ```console POST my-index-000001/_forcemerge ``` +% TEST[setup:my_index] ### Shrink an existing index to fewer shards [shrink-existing-index-to-fewer-shards] @@ -364,6 +384,7 @@ If you encounter this problem, try to mitigate it by using the [Force Merge API] ```console POST my-index-000001/_forcemerge?only_expunge_deletes=true ``` +% TEST[setup:my_index] This will launch an asynchronous task which can be monitored via the [Task Management API](https://www.elastic.co/docs/api/doc/elasticsearch/group/endpoint-tasks). diff --git a/deploy-manage/remote-clusters/remote-clusters-api-key.md b/deploy-manage/remote-clusters/remote-clusters-api-key.md index 73174d25e5..6dd62556a8 100644 --- a/deploy-manage/remote-clusters/remote-clusters-api-key.md +++ b/deploy-manage/remote-clusters/remote-clusters-api-key.md @@ -167,6 +167,8 @@ PUT /_cluster/settings } } ``` +% TEST[setup:host] +% TEST[s/127.0.0.1:{{remote-interface-default-port}}/${transport_host}/] 1. The cluster alias of this remote cluster is `cluster_one`. 2. Specifies the hostname and remote cluster port of a seed node in the remote cluster. @@ -177,6 +179,7 @@ You can use the [remote cluster info API](https://www.elastic.co/docs/api/doc/el ```console GET /_remote/info ``` +% TEST[continued] The API response indicates that the local cluster is connected to the remote cluster with the cluster alias `cluster_one`: @@ -196,6 +199,10 @@ The API response indicates that the local cluster is connected to the remote clu } } ``` +% TESTRESPONSE[s/127.0.0.1:{{remote-interface-default-port}}/$body.cluster_one.seeds.0/] +% TESTRESPONSE[s/ifeval::(.|\n)*endif::[]//] +% TEST[s/"connected" : true/"connected" : $body.cluster_one.connected/] +% TEST[s/"num_nodes_connected" : 1/"num_nodes_connected" : $body.cluster_one.num_nodes_connected/] 1. The number of nodes in the remote cluster the local cluster is connected to. 2. Indicates whether to skip the remote cluster if searched through {{ccs}} but no nodes are available. @@ -238,6 +245,10 @@ PUT _cluster/settings } } ``` +% TEST[setup:host] +% TEST[s/127.0.0.1:{{remote-interface-default-port}}/${transport_host}/] +% TEST[s/{{remote-interface-default-port-plus1}}/9301/] +% TEST[s/{{remote-interface-default-port-plus2}}/9302/] You can dynamically update settings for a remote cluster after the initial configuration. The following request updates the compression settings for `cluster_two`, and the compression and ping schedule settings for `cluster_three`. @@ -264,6 +275,7 @@ PUT _cluster/settings } } ``` +% TEST[continued] You can delete a remote cluster from the cluster settings by passing `null` values for each remote cluster setting. The following request removes `cluster_two` from the cluster settings, leaving `cluster_one` and `cluster_three` intact: @@ -284,6 +296,7 @@ PUT _cluster/settings } } ``` +% TEST[continued] ### Statically configure remote clusters [_statically_configure_remote_clusters] @@ -355,6 +368,7 @@ POST /_security/role/remote-replication ] } ``` +% TEST[skip:TODO] After creating the local `remote-replication` role, use the [Create or update users](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-security-put-user) API to create a user on the local cluster cluster and assign the `remote-replication` role. For example, the following request assigns the `remote-replication` role to a user named `cross-cluster-user`: @@ -365,6 +379,7 @@ POST /_security/user/cross-cluster-user "roles" : [ "remote-replication" ] } ``` +% TEST[skip:TODO] Note that you only need to create this user on the local cluster. @@ -391,6 +406,7 @@ POST /_security/role/remote-search ] } ``` +% TEST[skip:TODO] After creating the `remote-search` role, use the [Create or update users](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-security-put-user) API to create a user on the local cluster and assign the `remote-search` role. For example, the following request assigns the `remote-search` role to a user named `cross-search-user`: @@ -401,5 +417,6 @@ POST /_security/user/cross-search-user "roles" : [ "remote-search" ] } ``` +% TEST[skip:TODO] Note that you only need to create this user on the local cluster. diff --git a/deploy-manage/remote-clusters/remote-clusters-cert.md b/deploy-manage/remote-clusters/remote-clusters-cert.md index ba751510aa..738433afe6 100644 --- a/deploy-manage/remote-clusters/remote-clusters-cert.md +++ b/deploy-manage/remote-clusters/remote-clusters-cert.md @@ -85,6 +85,8 @@ PUT /_cluster/settings } } ``` +% TEST[setup:host] +% TEST[s/127.0.0.1:{{remote-interface-default-port}}/${transport_host}/] 1. The cluster alias of this remote cluster is `cluster_one`. 2. Specifies the hostname and transport port of a seed node in the remote cluster. @@ -95,6 +97,7 @@ You can use the [remote cluster info API](https://www.elastic.co/docs/api/doc/el ```console GET /_remote/info ``` +% TEST[continued] The API response indicates that the local cluster is connected to the remote cluster with the cluster alias `cluster_one`: @@ -113,6 +116,10 @@ The API response indicates that the local cluster is connected to the remote clu } } ``` +% TESTRESPONSE[s/127.0.0.1:{{remote-interface-default-port}}/$body.cluster_one.seeds.0/] +% TESTRESPONSE[s/ifeval::(.|\n)*endif::[]//] +% TEST[s/"connected" : true/"connected" : $body.cluster_one.connected/] +% TEST[s/"num_nodes_connected" : 1/"num_nodes_connected" : $body.cluster_one.num_nodes_connected/] 1. The number of nodes in the remote cluster the local cluster is connected to. 2. Indicates whether to skip the remote cluster if searched through {{ccs}} but no nodes are available. @@ -154,6 +161,10 @@ PUT _cluster/settings } } ``` +% TEST[setup:host] +% TEST[s/127.0.0.1:{{remote-interface-default-port}}/${transport_host}/] +% TEST[s/{{remote-interface-default-port-plus1}}/9301/] +% TEST[s/{{remote-interface-default-port-plus2}}/9302/] You can dynamically update settings for a remote cluster after the initial configuration. The following request updates the compression settings for `cluster_two`, and the compression and ping schedule settings for `cluster_three`. @@ -180,6 +191,7 @@ PUT _cluster/settings } } ``` +% TEST[continued] You can delete a remote cluster from the cluster settings by passing `null` values for each remote cluster setting. The following request removes `cluster_two` from the cluster settings, leaving `cluster_one` and `cluster_three` intact: @@ -200,6 +212,7 @@ PUT _cluster/settings } } ``` +% TEST[continued] ### Statically configure remote clusters [_statically_configure_remote_clusters_2] @@ -327,6 +340,7 @@ POST /_security/user/cross-cluster-user "roles" : [ "remote-replication" ] } ``` +% TEST[continued] ::::{note} You only need to create this user on the **local** cluster. @@ -395,6 +409,7 @@ POST /_security/user/cross-search-user "roles" : [ "remote-search" ] } ``` +% TEST[continued] ::::{note} You only need to create this user on the **local** cluster. diff --git a/deploy-manage/tools/snapshot-and-restore/azure-repository.md b/deploy-manage/tools/snapshot-and-restore/azure-repository.md index f193a105f2..079e83972a 100644 --- a/deploy-manage/tools/snapshot-and-restore/azure-repository.md +++ b/deploy-manage/tools/snapshot-and-restore/azure-repository.md @@ -58,6 +58,7 @@ PUT _snapshot/my_backup } } ``` +% TEST[skip:we don’t have azure setup while testing this] If you are using the `default` client, you may omit the `client` repository setting: @@ -67,6 +68,7 @@ PUT _snapshot/my_backup "type": "azure" } ``` +% TEST[skip:we don’t have azure setup while testing this] ::::{note} In progress snapshot or restore jobs will not be preempted by a **reload** of the storage secure settings. They will complete using the client as it was built when the operation started. @@ -143,6 +145,7 @@ PUT _snapshot/my_backup } } ``` +% TEST[skip:we don’t have azure setup while testing this] `client` : The name of the Azure repository client to use. Defaults to `default`. diff --git a/deploy-manage/tools/snapshot-and-restore/create-snapshots.md b/deploy-manage/tools/snapshot-and-restore/create-snapshots.md index 0a1de5d642..eb208ad15c 100644 --- a/deploy-manage/tools/snapshot-and-restore/create-snapshots.md +++ b/deploy-manage/tools/snapshot-and-restore/create-snapshots.md @@ -90,6 +90,7 @@ POST _security/role/slm-admin ] } ``` +% TEST[skip:security is not enabled here] To grant read-only access to {{slm-init}} policies and the snapshot history, you can set up a role with the `read_slm` cluster privilege and read access to the {{slm}} history indices. @@ -107,6 +108,7 @@ POST _security/role/slm-read-only ] } ``` +% TEST[skip:security is not enabled here] ### Create an {{slm-init}} policy [create-slm-policy] @@ -153,6 +155,7 @@ To run a policy in {{kib}}, go to the **Policies** page and click the run icon u ```console POST _slm/policy/nightly-snapshots/_execute ``` +% TEST[skip:we can’t easily handle snapshots from docs tests] The snapshot process runs in the background. To monitor its progress, see [Monitor a snapshot](#monitor-snapshot). @@ -194,6 +197,7 @@ To take a snapshot without an {{slm-init}} policy, use the [create snapshot API] # PUT _snapshot/my_repository/ PUT _snapshot/my_repository/%3Cmy_snapshot_%7Bnow%2Fd%7D%3E ``` +% TEST[s/3E/3E?wait_for_completion=true/] Depending on its size, a snapshot can take a while to complete. By default, the create snapshot API only initiates the snapshot process, which runs in the background. To block the client until the snapshot finishes, set the `wait_for_completion` query parameter to `true`. @@ -245,6 +249,7 @@ To delete a snapshot in {{kib}}, go to the **Snapshots** page and click the tras ```console DELETE _snapshot/my_repository/my_snapshot_2099.05.06 ``` +% TEST[setup:setup-snapshots] If you delete a snapshot that’s in progress, {{es}} cancels it. The snapshot process halts and deletes any files created for the snapshot. Deleting a snapshot doesn’t delete files used by other snapshots. @@ -292,6 +297,7 @@ The API returns: ] } ``` +% TESTRESPONSE[skip:response may vary based on features in test cluster] To include a specific feature state in a snapshot, specify the feature `name` in the `feature_states` array. @@ -347,6 +353,7 @@ PUT _slm/policy/nightly-cluster-state-snapshots } } ``` +% TEST[s/my_secure_repository/my_repository/] 1. Includes the cluster state. This also includes all feature states by default. 2. Excludes regular data streams and indices. diff --git a/deploy-manage/tools/snapshot-and-restore/google-cloud-storage-repository.md b/deploy-manage/tools/snapshot-and-restore/google-cloud-storage-repository.md index 29b42f60f1..cfb98d9926 100644 --- a/deploy-manage/tools/snapshot-and-restore/google-cloud-storage-repository.md +++ b/deploy-manage/tools/snapshot-and-restore/google-cloud-storage-repository.md @@ -69,6 +69,7 @@ A JSON service account file looks like this: "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/your-bucket@your-project-id.iam.gserviceaccount.com" } ``` +% NOTCONSOLE To provide this file to the repository, it must be stored in the [Elasticsearch keystore](../../security/secure-settings.md). You must add a `file` setting with the name `gcs.client.NAME.credentials_file` using the `add-file` subcommand. `NAME` is the name of the client configuration for the repository. The implicit client name is `default`, but a different client name can be specified in the repository settings with the `client` key. @@ -89,6 +90,8 @@ PUT _snapshot/my_gcs_repository } } ``` +% TEST[skip:we don’t have gcs setup while testing this] +% TEST[skip:we don’t have gcs setup while testing this] The `credentials_file` settings are [reloadable](../../security/secure-settings.md#reloadable-secure-settings). You can define these settings before the node is started, or call the [Nodes reload secure settings API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-nodes-reload-secure-settings) after the settings are defined to apply them to a running node. @@ -170,6 +173,7 @@ PUT _snapshot/my_gcs_repository } } ``` +% TEST[skip:we don’t have gcs set up while testing this] The following settings are supported: diff --git a/deploy-manage/tools/snapshot-and-restore/read-only-url-repository.md b/deploy-manage/tools/snapshot-and-restore/read-only-url-repository.md index 7eb5d438ee..8db5806147 100644 --- a/deploy-manage/tools/snapshot-and-restore/read-only-url-repository.md +++ b/deploy-manage/tools/snapshot-and-restore/read-only-url-repository.md @@ -26,6 +26,7 @@ PUT _snapshot/my_read_only_url_repository } } ``` +% TEST[skip:no access to url file path] ## Repository settings [read-only-url-repository-settings] diff --git a/deploy-manage/tools/snapshot-and-restore/restore-snapshot.md b/deploy-manage/tools/snapshot-and-restore/restore-snapshot.md index 264a42f432..6e4ac29560 100644 --- a/deploy-manage/tools/snapshot-and-restore/restore-snapshot.md +++ b/deploy-manage/tools/snapshot-and-restore/restore-snapshot.md @@ -58,11 +58,13 @@ You can also use the get repository API and the get snapshot API to find snapsho ```console GET _snapshot ``` +% TEST[setup:setup-snapshots] Then use the get snapshot API to get a list of snapshots in a specific repository. This also returns each snapshot’s contents. ```console GET _snapshot/my_repository/*?verbose=false ``` +% TEST[setup:setup-snapshots] ## Restore an index or data stream [restore-index-data-stream] @@ -90,6 +92,8 @@ DELETE my-index # Delete a data stream DELETE _data_stream/logs-my_app-default ``` +% TEST[setup:setup-snapshots] +% TEST[setup:setup-snapshots] In the restore request, explicitly specify any indices and data streams to restore. ```console @@ -98,6 +102,8 @@ POST _snapshot/my_repository/my_snapshot_2099.05.06/_restore "indices": "my-index,logs-my_app-default" } ``` +% TEST[continued] +% TEST[s/_restore/_restore?wait_for_completion=true/] ### Rename on restore [rename-on-restore] @@ -115,6 +121,8 @@ POST _snapshot/my_repository/my_snapshot_2099.05.06/_restore "rename_replacement": "restored-$1" } ``` +% TEST[setup:setup-snapshots] +% TEST[s/_restore/_restore?wait_for_completion=true/] If the rename options produce two or more indices or data streams with the same name, the restore operation fails. @@ -152,6 +160,7 @@ POST _reindex } } ``` +% TEST[continued] ## Restore a feature state [restore-feature-state] @@ -165,6 +174,7 @@ To view a snapshot’s feature states, use the get snapshot API. ```console GET _snapshot/my_repository/my_snapshot_2099.05.06 ``` +% TEST[setup:setup-snapshots] The response’s `feature_states` property contains a list of features in the snapshot as well as each feature’s indices. @@ -188,6 +198,10 @@ POST _snapshot/my_repository/my_snapshot_2099.05.06/_restore "indices": "-*" <2> } ``` +% TEST[setup:setup-snapshots] +% TEST[s/^/DELETE my-index\nDELETE _data_stream/logs-my_app-default\n/] +% TEST[s/_restore/_restore?wait_for_completion=true/] +% TEST[s/"feature_states": [ "geoip" ],//] 1. Exclude the cluster state from the restore operation. 2. Exclude the other indices and data streams in the snapshot from the restore operation. @@ -414,6 +428,7 @@ To get detailed information about ongoing shard recoveries, use the [index recov ```console GET my-index/_recovery ``` +% TEST[setup:setup-snapshots] To view any unassigned shards, use the [cat shards API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-cat-shards). @@ -434,6 +449,9 @@ GET _cluster/allocation/explain "current_node": "my-node" } ``` +% TEST[s/^/PUT my-index\n/] +% TEST[s/"primary": false,/"primary": false/] +% TEST[s/"current_node": "my-node"//] ## Cancel a restore [cancel-restore] diff --git a/deploy-manage/tools/snapshot-and-restore/s3-repository.md b/deploy-manage/tools/snapshot-and-restore/s3-repository.md index 920eebbf64..aba86c8544 100644 --- a/deploy-manage/tools/snapshot-and-restore/s3-repository.md +++ b/deploy-manage/tools/snapshot-and-restore/s3-repository.md @@ -31,6 +31,7 @@ PUT _snapshot/my_s3_repository } } ``` +% TEST[skip:we don’t have s3 setup while testing this] ## Client settings [repository-s3-client] @@ -47,6 +48,7 @@ PUT _snapshot/my_s3_repository } } ``` +% TEST[skip:we don’t have S3 setup while testing this] Most client settings can be added to the `elasticsearch.yml` configuration file with the exception of the secure settings, which you add to the {{es}} keystore. For more information about creating and updating the {{es}} keystore, see [Secure settings](../../security/secure-settings.md). @@ -153,6 +155,7 @@ PUT _snapshot/my_s3_repository } } ``` +% TEST[skip:we don’t have S3 set up while testing this] The following settings are supported: @@ -251,6 +254,7 @@ PUT _snapshot/my_s3_repository } } ``` +% TEST[skip:we don’t have s3 set up while testing this] This sets up a repository that uses all client settings from the client `my_client_name` except for the `endpoint` that is overridden to `my.s3.endpoint` by the repository settings. ` @@ -306,6 +310,7 @@ In order to restrict the Elasticsearch snapshot process to the minimum required "Version": "2012-10-17" } ``` +% NOTCONSOLE You may further restrict the permissions by specifying a prefix within the bucket, in this example, named "foo". @@ -348,6 +353,7 @@ You may further restrict the permissions by specifying a prefix within the bucke "Version": "2012-10-17" } ``` +% NOTCONSOLE The bucket needs to exist to register a repository for snapshots. If you did not create the bucket then the repository registration will fail. @@ -402,6 +408,7 @@ PUT /_cluster/settings } } ``` +% TEST[skip:we don’t really want to change this logger] Collect the Elasticsearch logs covering the time period of the failed analysis from all nodes in your cluster and share them with the supplier of your storage system along with the analysis response so they can use them to determine the problem. See the [AWS Java SDK](https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/java-dg-logging.html) documentation for further information, including details about other loggers that can be used to obtain even more verbose logs. When you have finished collecting the logs needed by your supplier, set the logger settings back to `null` to return to the default logging configuration and disable insecure network trace logging again. See [Logger](elasticsearch://reference/elasticsearch/configuration-reference/miscellaneous-cluster-settings.md#cluster-logger) and [Cluster update settings](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-cluster-put-settings) for more information. diff --git a/deploy-manage/tools/snapshot-and-restore/self-managed.md b/deploy-manage/tools/snapshot-and-restore/self-managed.md index 04bc31ca50..49ca575d68 100644 --- a/deploy-manage/tools/snapshot-and-restore/self-managed.md +++ b/deploy-manage/tools/snapshot-and-restore/self-managed.md @@ -83,12 +83,16 @@ PUT _snapshot/my_unverified_backup?verify=false } } ``` +% TEST[setup:setup-repository] +% TEST[s/my_unverified_backup_location/my_repository/] If wanted, you can manually run the repository verification check. To verify a repository in {{kib}}, go to the **Repositories** list page and click the name of a repository. Then click **Verify repository**. You can also use the [verify snapshot repository API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-snapshot-verify-repository). ```console POST _snapshot/my_unverified_backup/_verify ``` +% TEST[continued] +% TEST[s/my_unverified_backup_location/my_repository/] If successful, the request returns a list of nodes used to verify the repository. If verification fails, the request returns an error. @@ -106,6 +110,7 @@ You can also use the [clean up snapshot repository API](https://www.elastic.co/d ```console POST _snapshot/my_repository/_cleanup ``` +% TEST[setup:setup-snapshots] The API returns: @@ -117,6 +122,8 @@ The API returns: } } ``` +% TESTRESPONSE[s/"deleted_bytes": 20/"deleted_bytes": $body.results.deleted_bytes/] +% TESTRESPONSE[s/"deleted_blobs": 5/"deleted_blobs": $body.results.deleted_bytes/] Depending on the concrete repository implementation the numbers shown for bytes free as well as the number of blobs removed will either be an approximation or an exact result. Any non-zero value for the number of blobs removed implies that unreferenced blobs were found and subsequently cleaned up. diff --git a/deploy-manage/tools/snapshot-and-restore/shared-file-system-repository.md b/deploy-manage/tools/snapshot-and-restore/shared-file-system-repository.md index c15e3b26fc..5cf7bf1b42 100644 --- a/deploy-manage/tools/snapshot-and-restore/shared-file-system-repository.md +++ b/deploy-manage/tools/snapshot-and-restore/shared-file-system-repository.md @@ -42,6 +42,7 @@ PUT _snapshot/my_fs_backup } } ``` +% TEST[skip:no access to path] If you specify a relative path, {{es}} resolves the path using the first value in the `path.repo` setting. @@ -54,6 +55,7 @@ PUT _snapshot/my_fs_backup } } ``` +% TEST[skip:no access to path] 1. The first value in the `path.repo` setting is `/mount/backups`. This relative path, `my_fs_backup_location`, resolves to `/mount/backups/my_fs_backup_location`. @@ -74,6 +76,8 @@ PUT _snapshot/my_fs_backup } } ``` +% TEST[skip:no access to path] +:::::: :::::: ::::::{tab-item} Windows @@ -101,6 +105,7 @@ PUT _snapshot/my_fs_backup } } ``` +% TEST[skip:no access to path] If you specify a relative path, {{es}} resolves the path using the first value in the `path.repo` setting. @@ -113,6 +118,7 @@ PUT _snapshot/my_fs_backup } } ``` +% TEST[skip:no access to path] 1. The first value in the `path.repo` setting is `E:\Mount\Backups`. This relative path, `My_fs_backup_location`, resolves to `E:\Mount\Backups\My_fs_backup_location`. diff --git a/deploy-manage/tools/snapshot-and-restore/source-only-repository.md b/deploy-manage/tools/snapshot-and-restore/source-only-repository.md index 523d63bad8..9733db8ec6 100644 --- a/deploy-manage/tools/snapshot-and-restore/source-only-repository.md +++ b/deploy-manage/tools/snapshot-and-restore/source-only-repository.md @@ -37,6 +37,8 @@ PUT _snapshot/my_src_only_repository } } ``` +% TEST[setup:setup-repository] +% TEST[s/my_backup_repository/my_repository/] ## Repository settings [source-only-repository-settings] diff --git a/deploy-manage/upgrade/deployment-or-cluster/archived-settings.md b/deploy-manage/upgrade/deployment-or-cluster/archived-settings.md index 63d27ef8d1..0afae8816c 100644 --- a/deploy-manage/upgrade/deployment-or-cluster/archived-settings.md +++ b/deploy-manage/upgrade/deployment-or-cluster/archived-settings.md @@ -48,6 +48,7 @@ Use the following [get index settings](https://www.elastic.co/docs/api/doc/elast ```console GET */_settings?flat_settings=true&filter_path=**.settings.archived* ``` +% TEST[s/^/PUT my-index\n/] To remove any archived index settings, use the following [indices update settings](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-put-settings) request. @@ -57,4 +58,5 @@ PUT /my-index/_settings "archived.*": null } ``` +% TEST[s/^/PUT my-index\n/] diff --git a/deploy-manage/users-roles/cluster-or-deployment-auth/built-in-sm.md b/deploy-manage/users-roles/cluster-or-deployment-auth/built-in-sm.md index 4dcfabf8c7..a96b7646c9 100644 --- a/deploy-manage/users-roles/cluster-or-deployment-auth/built-in-sm.md +++ b/deploy-manage/users-roles/cluster-or-deployment-auth/built-in-sm.md @@ -44,6 +44,7 @@ POST /_security/user/user1/_password "password" : "new-test-password" } ``` +% TEST[continued] ## Using the `user` API [native-users-api] diff --git a/deploy-manage/users-roles/cluster-or-deployment-auth/granting-privileges-for-data-streams-aliases.md b/deploy-manage/users-roles/cluster-or-deployment-auth/granting-privileges-for-data-streams-aliases.md index 34c79bb25b..9fcdda1608 100644 --- a/deploy-manage/users-roles/cluster-or-deployment-auth/granting-privileges-for-data-streams-aliases.md +++ b/deploy-manage/users-roles/cluster-or-deployment-auth/granting-privileges-for-data-streams-aliases.md @@ -27,18 +27,23 @@ A user is granted the `read` privilege to `my-data-stream`. "privileges" : [ "read" ] } ``` +% NOTCONSOLE Because the user is automatically granted the same privileges to the stream’s backing indices, the user can retrieve a document directly from `.ds-my-data-stream-2099.03.08-000002`: ```console GET .ds-my-data-stream-2099.03.08-000002/_doc/2 ``` +% TEST[continued] +% TEST[s/.ds-my-data-stream-2099.03.08-000002/my-index/] Later `my-data-stream` [rolls over](../../../manage-data/data-store/data-streams/use-data-stream.md#manually-roll-over-a-data-stream). This creates a new backing index: `.ds-my-data-stream-2099.03.09-000003`. Because the user still has the `read` privilege for `my-data-stream`, the user can retrieve documents directly from `.ds-my-data-stream-2099.03.09-000003`: ```console GET .ds-my-data-stream-2099.03.09-000003/_doc/2 ``` +% TEST[continued] +% TEST[s/.ds-my-data-stream-2099.03.09-000003/my-index/] ## Alias privileges [index-alias-privileges] @@ -58,12 +63,14 @@ For example, the `current_year` alias points only to the `2015` index. A user is "privileges" : [ "read" ] } ``` +% NOTCONSOLE When the user attempts to retrieve a document from the `current_year` alias, {{es}} rejects the request. ```console GET current_year/_doc/1 ``` +% TEST[s/^/PUT 2015\n{"aliases": {"current_year": {}}}\nPUT 2015/_doc/1\n{}\n/] To retrieve documents from `current_year`, the user must have the `read` index privilege for the alias. @@ -73,5 +80,6 @@ To retrieve documents from `current_year`, the user must have the `read` index p "privileges" : [ "read" ] } ``` +% NOTCONSOLE diff --git a/deploy-manage/users-roles/cluster-or-deployment-auth/role-mapping-resources.md b/deploy-manage/users-roles/cluster-or-deployment-auth/role-mapping-resources.md index b4d77b9f1a..b4c89b2172 100644 --- a/deploy-manage/users-roles/cluster-or-deployment-auth/role-mapping-resources.md +++ b/deploy-manage/users-roles/cluster-or-deployment-auth/role-mapping-resources.md @@ -80,6 +80,7 @@ The `groups` field is multi-valued; a user can belong to many groups. When a `fi ```js { "field" : { "groups" : "admin" } } ``` +% NOTCONSOLE For additional realm-specific details, see [Active Directory and LDAP realms](../../../deploy-manage/users-roles/cluster-or-deployment-auth/mapping-users-groups-to-roles.md#ldap-role-mapping). diff --git a/deploy-manage/users-roles/cluster-or-deployment-auth/service-accounts.md b/deploy-manage/users-roles/cluster-or-deployment-auth/service-accounts.md index 5e0bbdf3ef..285e4dc8a1 100644 --- a/deploy-manage/users-roles/cluster-or-deployment-auth/service-accounts.md +++ b/deploy-manage/users-roles/cluster-or-deployment-auth/service-accounts.md @@ -74,6 +74,7 @@ To use a service account token, include the generated token value in a request w ```shell curl -H "Authorization: Bearer AAEAAWVsYXN0aWM...vZmxlZXQtc2VydmVyL3Rva2VuMTo3TFdaSDZ" http://localhost:9200/_security/_authenticate ``` +% NOTCONSOLE A successful authentication response includes a `token` field, which contains a `name` field for the name of the service token and a `type` field for the type of the service token: @@ -102,6 +103,7 @@ A successful authentication response includes a `token` field, which contains a "authentication_type": "token" } ``` +% NOTCONSOLE 1. Name of the service account token. 2. Type of service account token. The value always begins with `_service_account_` and is followed by a string that indicates the service token backend in use (can be either `file` or `index`). diff --git a/deploy-manage/users-roles/cluster-or-deployment-auth/token-based-authentication-services.md b/deploy-manage/users-roles/cluster-or-deployment-auth/token-based-authentication-services.md index 67db6261b3..d16dc93508 100644 --- a/deploy-manage/users-roles/cluster-or-deployment-auth/token-based-authentication-services.md +++ b/deploy-manage/users-roles/cluster-or-deployment-auth/token-based-authentication-services.md @@ -23,6 +23,7 @@ To use a service account token, include the generated token value in a request w ```shell curl -H "Authorization: Bearer AAEAAWVsYXN0aWMvZ...mXQtc2VydmMTpyNXdkYmRib1FTZTl2R09Ld2FKR0F3" http://localhost:9200/_cluster/health ``` +% NOTCONSOLE ::::{important} Do not attempt to use service accounts for authenticating individual users. Service accounts can only be authenticated with service tokens, which are not applicable to regular users. diff --git a/explore-analyze/alerts-cases/watcher/actions-email.md b/explore-analyze/alerts-cases/watcher/actions-email.md index ab7a52eb97..f11ae4c623 100644 --- a/explore-analyze/alerts-cases/watcher/actions-email.md +++ b/explore-analyze/alerts-cases/watcher/actions-email.md @@ -32,6 +32,7 @@ For example, the following email action uses a template to include data from the } } ``` +% NOTCONSOLE 1. The id of the action. 2. The action type is set to `email`. @@ -74,6 +75,7 @@ To configure attachments, specify a name for the attached file and the type of a } } ``` +% NOTCONSOLE 1. The ID of the attachment, which is used as the file name in the email attachment. 2. The type of the attachment and its specific configuration. diff --git a/explore-analyze/alerts-cases/watcher/actions-index.md b/explore-analyze/alerts-cases/watcher/actions-index.md index 14d52df6a5..b77ac118b0 100644 --- a/explore-analyze/alerts-cases/watcher/actions-index.md +++ b/explore-analyze/alerts-cases/watcher/actions-index.md @@ -27,6 +27,7 @@ The following snippet shows a simple `index` action definition: } } ``` +% NOTCONSOLE 1. The id of the action 2. An optional [condition](condition.md) to restrict action execution @@ -78,6 +79,7 @@ The following snippet shows a multi-document `index` action definition: } } ``` +% NOTCONSOLE 1. The document’s index 2. An optional `_id` for the document diff --git a/explore-analyze/alerts-cases/watcher/actions-jira.md b/explore-analyze/alerts-cases/watcher/actions-jira.md index dd395e5d77..96fe39f386 100644 --- a/explore-analyze/alerts-cases/watcher/actions-jira.md +++ b/explore-analyze/alerts-cases/watcher/actions-jira.md @@ -42,6 +42,7 @@ The following snippet shows a simple jira action definition: } } ``` +% NOTCONSOLE 1. The name of a Jira account configured in `elasticsearch.yml`. 2. The key of the Jira project in which the issue will be created. diff --git a/explore-analyze/alerts-cases/watcher/actions-logging.md b/explore-analyze/alerts-cases/watcher/actions-logging.md index e973f4657b..56dff578f9 100644 --- a/explore-analyze/alerts-cases/watcher/actions-logging.md +++ b/explore-analyze/alerts-cases/watcher/actions-logging.md @@ -29,6 +29,7 @@ The following snippet shows a simple logging action definition: } } ``` +% NOTCONSOLE 1. The id of the action. 2. An optional [transform](transform.md) to transform the payload before executing the `logging` action. diff --git a/explore-analyze/alerts-cases/watcher/actions-pagerduty.md b/explore-analyze/alerts-cases/watcher/actions-pagerduty.md index 7cae701c35..fc78727205 100644 --- a/explore-analyze/alerts-cases/watcher/actions-pagerduty.md +++ b/explore-analyze/alerts-cases/watcher/actions-pagerduty.md @@ -28,6 +28,7 @@ The following snippet shows a simple PagerDuty action definition: } } ``` +% NOTCONSOLE 1. Description of the message @@ -59,6 +60,7 @@ To give the PagerDuty incident some more context, you can attach the payload as } } ``` +% NOTCONSOLE ## Pagerduty action attributes [pagerduty-action-attributes] diff --git a/explore-analyze/alerts-cases/watcher/actions-slack.md b/explore-analyze/alerts-cases/watcher/actions-slack.md index 0770c0dd78..9191c80db6 100644 --- a/explore-analyze/alerts-cases/watcher/actions-slack.md +++ b/explore-analyze/alerts-cases/watcher/actions-slack.md @@ -31,6 +31,7 @@ The following snippet shows a simple slack action definition: } } ``` +% NOTCONSOLE 1. The channels and users you want to send the message to. 2. The content of the message. @@ -63,6 +64,7 @@ The following snippet shows a standard message attachment: } } ``` +% NOTCONSOLE ## Dynamic attachments [slack-dynamic-attachment] @@ -119,6 +121,7 @@ In the following example, the watch input executes a search with a date histogra } } ``` +% NOTCONSOLE 1. The list generated by the action’s transform. 2. The parameter placeholders refer to attributes in each item of the list generated by the transform. diff --git a/explore-analyze/alerts-cases/watcher/actions-webhook.md b/explore-analyze/alerts-cases/watcher/actions-webhook.md index 3c6674f245..2a0725d787 100644 --- a/explore-analyze/alerts-cases/watcher/actions-webhook.md +++ b/explore-analyze/alerts-cases/watcher/actions-webhook.md @@ -32,6 +32,7 @@ The following snippet shows a simple webhook action definition: } } ``` +% NOTCONSOLE 1. The id of the action 2. An optional [transform](transform.md) to transform the payload before executing the `webhook` action @@ -64,6 +65,7 @@ You can use basic authentication when sending a request to a secured webservice. } } ``` +% NOTCONSOLE 1. The username and password for the user creating the issue @@ -92,6 +94,7 @@ You can specify query parameters to send with the request with the `params` fiel } } ``` +% NOTCONSOLE 1. The parameter values can contain templated strings. @@ -115,6 +118,7 @@ You can specify request headers to send with the request with the `headers` fiel } } ``` +% NOTCONSOLE 1. The header values can contain templated strings. diff --git a/explore-analyze/alerts-cases/watcher/actions.md b/explore-analyze/alerts-cases/watcher/actions.md index b9f92e6c7a..329ed7c7a2 100644 --- a/explore-analyze/alerts-cases/watcher/actions.md +++ b/explore-analyze/alerts-cases/watcher/actions.md @@ -153,6 +153,7 @@ To acknowledge an action, you use the [ack watch API](https://www.elastic.co/doc ```console POST _watcher/watch//_ack/ ``` +% TEST[skip:https://github.com/elastic/x-plugins/issues/2513] Where `` is the id of the watch and `` is a comma-separated list of the action ids you want to acknowledge. To acknowledge all actions, omit the `actions` parameter. diff --git a/explore-analyze/alerts-cases/watcher/condition-always.md b/explore-analyze/alerts-cases/watcher/condition-always.md index 01552f8301..ec9e65fcf1 100644 --- a/explore-analyze/alerts-cases/watcher/condition-always.md +++ b/explore-analyze/alerts-cases/watcher/condition-always.md @@ -24,3 +24,4 @@ There are no attributes to specify for the `always` condition. To explicitly use "always" : {} } ``` +% NOTCONSOLE diff --git a/explore-analyze/alerts-cases/watcher/condition-array-compare.md b/explore-analyze/alerts-cases/watcher/condition-array-compare.md index d4c2453f1e..25282f38bd 100644 --- a/explore-analyze/alerts-cases/watcher/condition-array-compare.md +++ b/explore-analyze/alerts-cases/watcher/condition-array-compare.md @@ -31,6 +31,7 @@ For example, the following `array_compare` condition returns `true` if there is } } ``` +% NOTCONSOLE 1. The path to the array in the execution context that you want to evaluate, specified in dot notation. 2. The path to the field in each array element that you want to evaluate. diff --git a/explore-analyze/alerts-cases/watcher/condition-compare.md b/explore-analyze/alerts-cases/watcher/condition-compare.md index 8e371c8645..b01f6a5b9d 100644 --- a/explore-analyze/alerts-cases/watcher/condition-compare.md +++ b/explore-analyze/alerts-cases/watcher/condition-compare.md @@ -37,6 +37,7 @@ To use the `compare` condition, you specify the value in the execution context t } } ``` +% NOTCONSOLE 1. Use dot notation to reference a value in the execution context. 2. Specify a comparison operator and the value you want to compare against. @@ -54,6 +55,7 @@ You can also compare two values in the execution context by specifying the compa } } ``` +% NOTCONSOLE ## Using date math in a compare condition [compare-condition-date-math] @@ -70,6 +72,7 @@ When comparing dates and times, you can use date math expressions of the form `< } } ``` +% NOTCONSOLE ## Accessing values in the execution context [_accessing_values_in_the_execution_context] diff --git a/explore-analyze/alerts-cases/watcher/condition-never.md b/explore-analyze/alerts-cases/watcher/condition-never.md index 843c176e3c..f0750c6803 100644 --- a/explore-analyze/alerts-cases/watcher/condition-never.md +++ b/explore-analyze/alerts-cases/watcher/condition-never.md @@ -20,3 +20,4 @@ There are no attributes to specify for the `never` condition. To use the it, you "never" : {} } ``` +% NOTCONSOLE diff --git a/explore-analyze/alerts-cases/watcher/condition-script.md b/explore-analyze/alerts-cases/watcher/condition-script.md index 937517b579..e5b68d83f1 100644 --- a/explore-analyze/alerts-cases/watcher/condition-script.md +++ b/explore-analyze/alerts-cases/watcher/condition-script.md @@ -20,6 +20,7 @@ The following snippet configures an inline `script` condition that always return "script" : "return true" } ``` +% NOTCONSOLE This example defines a script as a simple string. This format is actually a shortcut for defining an [inline](#condition-script-inline) script. The formal definition of a script is an object that specifies the script type and optional language and parameter values. If the `lang` attribute is omitted, the language defaults to `painless`. Elasticsearch supports two types of scripts, [inline](#condition-script-inline) and [stored](#condition-script-stored). @@ -36,6 +37,7 @@ For example, the following snippet shows a formal definition of an `inline` scri } } ``` +% NOTCONSOLE ## Inline scripts [condition-script-inline] @@ -48,6 +50,7 @@ Inline scripts are scripts that are defined in the condition itself. The followi } } ``` +% NOTCONSOLE ## Stored scripts [condition-script-stored] @@ -60,6 +63,7 @@ Stored scripts refer to scripts that were [stored](../../scripting/modules-scrip } } ``` +% NOTCONSOLE As with [inline](#condition-script-inline) scripts, you can also specify the script language and parameters: @@ -72,6 +76,7 @@ As with [inline](#condition-script-inline) scripts, you can also specify the scr } } ``` +% NOTCONSOLE ## Accessing the watch payload [accessing-watch-payload] @@ -102,12 +107,14 @@ For example, the following snippet defines a watch that uses a [`search` input]( } } ``` +% NOTCONSOLE When you’re using a scripted condition to evaluate an Elasticsearch response, keep in mind that the fields in the response are no longer in their native data types. For example, the `@timestamp` in the response is a string, rather than a `DateTime`. To compare the response `@timestamp` against the `ctx.execution_time`, you need to parse the `@timestamp` string into a `ZonedDateTime`. For example: ```js java.time.ZonedDateTime.parse(@timestamp) ``` +% NOTCONSOLE You can reference the following variables in the watch context: diff --git a/explore-analyze/alerts-cases/watcher/how-watcher-works.md b/explore-analyze/alerts-cases/watcher/how-watcher-works.md index 14b8438259..2b676de3c2 100644 --- a/explore-analyze/alerts-cases/watcher/how-watcher-works.md +++ b/explore-analyze/alerts-cases/watcher/how-watcher-works.md @@ -127,6 +127,7 @@ PUT _watcher/settings "index.routing.allocation.include.role": "watcher" } ``` +% TEST[skip:indexes don’t assign] When the {{watcher}} service is stopped, the scheduler stops with it. Trigger engines use a separate thread pool from the one used to execute watches. @@ -204,6 +205,7 @@ The following snippet shows the basic structure of the *Watch Execution Context* "vars" : { ... } <6> } ``` +% NOTCONSOLE 1. Any static metadata specified in the watch definition. 2. The current watch payload. @@ -247,6 +249,7 @@ For example, the following snippet shows how templates enable dynamic subjects i } } ``` +% NOTCONSOLE #### Inline templates and scripts [inline-templates-scripts] @@ -263,6 +266,7 @@ To define an inline template or script, you simply specify it directly in the va } } ``` +% NOTCONSOLE For a script, you simply specify the inline script as the value of the `script` field. For example: @@ -271,6 +275,7 @@ For a script, you simply specify the inline script as the value of the `script` "script" : "return true" } ``` +% NOTCONSOLE You can also explicitly specify the inline type by using a formal object definition as the field value. For example: @@ -285,6 +290,7 @@ You can also explicitly specify the inline type by using a formal object definit } } ``` +% NOTCONSOLE The formal object definition for a script would be: @@ -295,6 +301,7 @@ The formal object definition for a script would be: } } ``` +% NOTCONSOLE #### Stored templates and scripts [stored-templates-scripts] @@ -320,4 +327,5 @@ To reference a stored script or template, you use the formal object definition a } } ``` +% NOTCONSOLE diff --git a/explore-analyze/alerts-cases/watcher/input-chain.md b/explore-analyze/alerts-cases/watcher/input-chain.md index 470e81a17b..ca85c75ec6 100644 --- a/explore-analyze/alerts-cases/watcher/input-chain.md +++ b/explore-analyze/alerts-cases/watcher/input-chain.md @@ -39,6 +39,7 @@ For example, the following chain input loads data from an HTTP server using the } } ``` +% NOTCONSOLE 1. The inputs in a chain are specified as an array to guarantee the order in which the inputs are processed. (JSON does not guarantee the order of arbitrary objects.) 2. Loads the `path` set by the `first` input. diff --git a/explore-analyze/alerts-cases/watcher/input-http.md b/explore-analyze/alerts-cases/watcher/input-http.md index 596500a22b..a296132fcc 100644 --- a/explore-analyze/alerts-cases/watcher/input-http.md +++ b/explore-analyze/alerts-cases/watcher/input-http.md @@ -32,6 +32,7 @@ To query an external Elasticsearch cluster, you specify the cluster’s `host` a } } ``` +% NOTCONSOLE You can use the full Elasticsearch [Query DSL](../../query-filter/languages/querydsl.md) to perform more sophisticated searches. For example, the following `http` input retrieves all documents that contain `event` in the `category` field: @@ -47,6 +48,7 @@ You can use the full Elasticsearch [Query DSL](../../query-filter/languages/quer } } ``` +% NOTCONSOLE ## Calling Elasticsearch APIs [_calling_elasticsearch_apis] @@ -66,6 +68,7 @@ To load the data from other Elasticsearch APIs, specify the API endpoint as the } } ``` +% NOTCONSOLE 1. Enabling this attribute returns the `bytes` values in the response in human readable format. @@ -90,6 +93,7 @@ You can use `http` input to get data from any external web service. The `http` i } } ``` +% NOTCONSOLE You can also pass in service-specific API keys and other information through the `params` attribute. For example, the following `http` input loads the current weather forecast for Amsterdam from the [OpenWeatherMap](http://openweathermap.org/appid) service: @@ -107,6 +111,7 @@ You can also pass in service-specific API keys and other information through the } } ``` +% NOTCONSOLE ### Using token-based authentication [_using_token_based_authentication] @@ -127,6 +132,7 @@ You can also call an API using a `Bearer token` instead of basic authentication. } } ``` +% NOTCONSOLE ## Using templates [_using_templates_2] @@ -146,6 +152,7 @@ For example, the following snippet uses templates to specify what index to query } } ``` +% NOTCONSOLE ## Accessing the HTTP response [_accessing_the_http_response] diff --git a/explore-analyze/alerts-cases/watcher/input-search.md b/explore-analyze/alerts-cases/watcher/input-search.md index 277e4fb03f..d7a7d5e482 100644 --- a/explore-analyze/alerts-cases/watcher/input-search.md +++ b/explore-analyze/alerts-cases/watcher/input-search.md @@ -33,6 +33,7 @@ For example, the following input retrieves all `event` documents from the `logs` } } ``` +% NOTCONSOLE You can use date math and wildcards when specifying indices. For example, the following input loads the latest VIXZ quote from today’s daily quotes index: @@ -56,6 +57,7 @@ You can use date math and wildcards when specifying indices. For example, the fo } } ``` +% NOTCONSOLE ## Extracting specific fields [_extracting_specific_fields] @@ -73,6 +75,7 @@ For example, the following input loads only the total number of hits into the wa } }, ``` +% NOTCONSOLE ## Using Templates [_using_templates] @@ -96,6 +99,7 @@ The `search` input supports [search templates](../../../solutions/search/search- ... } ``` +% NOTCONSOLE ## Applying conditions [_applying_conditions] @@ -119,6 +123,7 @@ The `search` input is often used in conjunction with the [`script`](condition-sc ... } ``` +% NOTCONSOLE ## Accessing the search results [_accessing_the_search_results] diff --git a/explore-analyze/alerts-cases/watcher/input-simple.md b/explore-analyze/alerts-cases/watcher/input-simple.md index 536aff8e03..c1b3e64a74 100644 --- a/explore-analyze/alerts-cases/watcher/input-simple.md +++ b/explore-analyze/alerts-cases/watcher/input-simple.md @@ -24,6 +24,7 @@ You can define the static data as a string (`str`), numeric value (`num`), or an } } ``` +% NOTCONSOLE For example, the following watch uses the `simple` input to set the recipient name for a daily reminder email: @@ -50,3 +51,4 @@ For example, the following watch uses the `simple` input to set the recipient na } } ``` +% NOTCONSOLE diff --git a/explore-analyze/alerts-cases/watcher/schedule-types.md b/explore-analyze/alerts-cases/watcher/schedule-types.md index 5b164ca40e..72c39c08e9 100644 --- a/explore-analyze/alerts-cases/watcher/schedule-types.md +++ b/explore-analyze/alerts-cases/watcher/schedule-types.md @@ -43,6 +43,7 @@ For example, the following `hourly` schedule triggers at minute 30 every hour-- } } ``` +% NOTCONSOLE ### Configuring a multiple times hourly schedule [_configuring_a_multiple_times_hourly_schedule] @@ -58,6 +59,7 @@ To configure an `hourly` schedule that triggers at multiple times during the hou } } ``` +% NOTCONSOLE @@ -86,6 +88,7 @@ To configure a once a day schedule, you specify a single time with the `at` attr } } ``` +% NOTCONSOLE ### Configuring a multiple times daily schedule [_configuring_a_multiple_times_daily_schedule] @@ -101,6 +104,7 @@ To configure a `daily` schedule that triggers at multiple times during the day, } } ``` +% NOTCONSOLE ### Specifying times using objects [specifying-times-using-objects] @@ -123,6 +127,7 @@ For example, the following `daily` schedule triggers once every day at 5:00 PM: } } ``` +% NOTCONSOLE To specify multiple times using the object notation, you specify multiple hours or minutes as an array. For example, following `daily` schedule triggers at `00:00`, `00:30`, `12:00`, `12:30`, `17:00` and `17:30` every day: @@ -140,6 +145,7 @@ To specify multiple times using the object notation, you specify multiple hours } } ``` +% NOTCONSOLE ### Specifying a time zone for a daily schedule [specifying-time-zone-for-daily-schedule] @@ -161,6 +167,7 @@ By default, daily schedules are evaluated in the UTC time zone. To use a differe } } ``` +% NOTCONSOLE @@ -190,6 +197,7 @@ To configure a once a week schedule, you specify the day with the `on` attribute } } ``` +% NOTCONSOLE ::::{note} You can also specify the day and time with the `day` and `time` attributes, they are interchangeable with `on` and `at`. @@ -213,6 +221,7 @@ To configure a `weekly` schedule that triggers multiple times a week, you can sp } } ``` +% NOTCONSOLE Alternatively, you can specify days and times in an object that has `on` and `minute` attributes that contain an array of values. For example, the following `weekly` schedule triggers every Tuesday and Friday at 12:00 PM and 17:00 PM: @@ -228,6 +237,7 @@ Alternatively, you can specify days and times in an object that has `on` and `mi } } ``` +% NOTCONSOLE @@ -248,6 +258,7 @@ By default, weekly schedules are evaluated in the UTC time zone. To use a differ } } ``` +% NOTCONSOLE ## {{watcher}} monthly schedule [schedule-monthly] @@ -270,6 +281,7 @@ To configure a once a month schedule, you specify a single day and time with the } } ``` +% NOTCONSOLE ::::{note} You can also specify the day and time with the `day` and `time` attributes, they are interchangeable with `on` and `at`. @@ -293,6 +305,7 @@ To configure a `monthly` schedule that triggers multiple times a month, you can } } ``` +% NOTCONSOLE Alternatively, you can specify days and times in an object that has `on` and `at` attributes that contain an array of values. For example, the following `monthly` schedule triggers at 12:00 AM and 12:00 PM on the 10th and 20th of each month. @@ -308,6 +321,7 @@ Alternatively, you can specify days and times in an object that has `on` and `at } } ``` +% NOTCONSOLE @@ -328,6 +342,7 @@ By default, monthly schedules are evaluated in the UTC time zone. To use a diffe } } ``` +% NOTCONSOLE ## {{watcher}} yearly schedule [schedule-yearly] @@ -356,6 +371,7 @@ To configure a once a year schedule, you specify the month with the `in` attribu } } ``` +% NOTCONSOLE ::::{note} You can also specify the month, day, and time with the `month`, `day`, and `time` attributes, they are interchangeable with `in`, `on`, and `at`. @@ -379,6 +395,7 @@ To configure a `yearly` schedule that triggers multiple times a year, you can sp } } ``` +% NOTCONSOLE Alternatively, you can specify the months, days, and times in an object that has `in`, `on`, and `minute` attributes that contain an array of values. For example, the following `yearly` schedule triggers at 12:00 AM and 12:00 PM on January 10th, January 20th, December 10th, and December 20th. @@ -395,6 +412,7 @@ Alternatively, you can specify the months, days, and times in an object that has } } ``` +% NOTCONSOLE @@ -416,6 +434,7 @@ By default, the `yearly` schedule is evaluated in the UTC time zone. To use a di } } ``` +% NOTCONSOLE ## {{watcher}} cron schedule [schedule-cron] @@ -443,6 +462,7 @@ To configure a `cron` schedule, you simply specify the cron expression as a stri ... } ``` +% NOTCONSOLE ### Configure a cron schedule with multiple times [_configuring_a_multiple_times_cron_schedule] @@ -463,6 +483,7 @@ To configure a `cron` schedule that triggers multiple times, you can specify an ... } ``` +% NOTCONSOLE @@ -484,6 +505,7 @@ By default, cron expressions are evaluated in the UTC time zone. To use a differ ... } ``` +% NOTCONSOLE ### Use croneval to validate cron expressions [croneval] @@ -530,6 +552,7 @@ For example, the following `interval` schedule triggers every five minutes: } } ``` +% NOTCONSOLE diff --git a/explore-analyze/alerts-cases/watcher/transform-chain.md b/explore-analyze/alerts-cases/watcher/transform-chain.md index d2c5da64bf..4c0036dcfe 100644 --- a/explore-analyze/alerts-cases/watcher/transform-chain.md +++ b/explore-analyze/alerts-cases/watcher/transform-chain.md @@ -35,6 +35,7 @@ You can use chain {{watcher-transforms}} to build more complex transforms out of ] } ``` +% NOTCONSOLE 1. The `chain` {{watcher-transform}} definition 2. The first transform in the chain (in this case, a `search` {{watcher-transform}}) diff --git a/explore-analyze/alerts-cases/watcher/transform-script.md b/explore-analyze/alerts-cases/watcher/transform-script.md index 83bafafb31..a0d1286ab3 100644 --- a/explore-analyze/alerts-cases/watcher/transform-script.md +++ b/explore-analyze/alerts-cases/watcher/transform-script.md @@ -22,6 +22,7 @@ The `script` {{watcher-transform}} is often useful when used in combination with } } ``` +% NOTCONSOLE 1. A simple `painless` script that creates a new payload with a single `time` field holding the scheduled time. diff --git a/explore-analyze/alerts-cases/watcher/transform-search.md b/explore-analyze/alerts-cases/watcher/transform-search.md index 1dd0d73bf5..88a281aaec 100644 --- a/explore-analyze/alerts-cases/watcher/transform-search.md +++ b/explore-analyze/alerts-cases/watcher/transform-search.md @@ -22,6 +22,7 @@ A [{{watcher-transform}}](transform.md) that executes a search on the cluster an } } ``` +% NOTCONSOLE Like every other search based construct, one can make use of the full search API supported by Elasticsearch. For example, the following search {{watcher-transform}} execute a search over all events indices, matching events with `error` priority: @@ -42,6 +43,7 @@ Like every other search based construct, one can make use of the full search API } } ``` +% NOTCONSOLE ## Transform search settings [transform-search-settings] @@ -95,6 +97,7 @@ For example, the following snippet shows a search that refers to the scheduled t } } ``` +% NOTCONSOLE The model of the template is a union between the provided `template.params` settings and the [standard watch execution context model](how-watcher-works.md#watch-execution-context). @@ -136,3 +139,4 @@ The following is an example of using templates that refer to provided parameters } } ``` +% NOTCONSOLE diff --git a/explore-analyze/alerts-cases/watcher/transform.md b/explore-analyze/alerts-cases/watcher/transform.md index 60a4a706c3..105a93597e 100644 --- a/explore-analyze/alerts-cases/watcher/transform.md +++ b/explore-analyze/alerts-cases/watcher/transform.md @@ -54,6 +54,7 @@ The following example defines two {{watcher-transforms}}, one at the watch level ] } ``` +% NOTCONSOLE 1. A watch level `transform` 2. An action level `transform` diff --git a/explore-analyze/alerts-cases/watcher/watch-cluster-status.md b/explore-analyze/alerts-cases/watcher/watch-cluster-status.md index 70edbdab55..499bf85f27 100644 --- a/explore-analyze/alerts-cases/watcher/watch-cluster-status.md +++ b/explore-analyze/alerts-cases/watcher/watch-cluster-status.md @@ -36,6 +36,7 @@ To get the status of your cluster, you can call the [cluster health API](https:/ ```console GET _cluster/health?pretty ``` +% TEST[continued] To load the health status into your watch, you simply add an [HTTP input](input-http.md) that calls the cluster health API: @@ -99,6 +100,7 @@ GET .watcher-history*/_search ] } ``` +% TEST[continued] ## Add a condition [health-add-condition] @@ -143,6 +145,8 @@ GET .watcher-history*/_search?pretty } } ``` +% TEST[continued] +% TEST[continued] ## Take action [health-take-action] @@ -229,3 +233,4 @@ To remove the watch, use the [delete watch API](https://www.elastic.co/docs/api/ ```console DELETE _watcher/watch/cluster_health_watch ``` +% TEST[continued] diff --git a/explore-analyze/alerts-cases/watcher/watcher-getting-started.md b/explore-analyze/alerts-cases/watcher/watcher-getting-started.md index 888f5b82aa..140a503e1c 100644 --- a/explore-analyze/alerts-cases/watcher/watcher-getting-started.md +++ b/explore-analyze/alerts-cases/watcher/watcher-getting-started.md @@ -56,6 +56,7 @@ GET .watcher-history*/_search?pretty ] } ``` +% TEST[continued] ## Add a condition [log-add-condition] @@ -98,6 +99,7 @@ POST logs/_doc "message": "Error: File not found" } ``` +% TEST[continued] Once you add this event, the next time the watch executes its condition will evaluate to `true`. The condition result is recorded as part of the `watch_record` each time the watch executes, so you can verify whether or not the condition was met by searching the watch history: @@ -114,6 +116,7 @@ GET .watcher-history*/_search?pretty } } ``` +% TEST[continued] ## Configure an action [log-take-action] @@ -159,6 +162,7 @@ To remove the watch, use the [delete watch API](https://www.elastic.co/docs/api/ ```console DELETE _watcher/watch/log_error_watch ``` +% TEST[continued] ## Required security privileges [required-security-privileges] diff --git a/explore-analyze/elastic-inference/inference-api/alibabacloud-ai-search-inference-integration.md b/explore-analyze/elastic-inference/inference-api/alibabacloud-ai-search-inference-integration.md index 9e6602661a..660b326be9 100644 --- a/explore-analyze/elastic-inference/inference-api/alibabacloud-ai-search-inference-integration.md +++ b/explore-analyze/elastic-inference/inference-api/alibabacloud-ai-search-inference-integration.md @@ -163,6 +163,7 @@ PUT _inference/completion/alibabacloud_ai_search_completion } } ``` +% TEST[skip:TBD] The next example shows how to create an {{infer}} endpoint called `alibabacloud_ai_search_rerank` to perform a `rerank` task type. @@ -178,6 +179,7 @@ PUT _inference/rerank/alibabacloud_ai_search_rerank } } ``` +% TEST[skip:TBD] The following example shows how to create an {{infer}} endpoint called `alibabacloud_ai_search_sparse` to perform a `sparse_embedding` task type. @@ -193,6 +195,7 @@ PUT _inference/sparse_embedding/alibabacloud_ai_search_sparse } } ``` +% TEST[skip:TBD] The following example shows how to create an {{infer}} endpoint called `alibabacloud_ai_search_embeddings` to perform a `text_embedding` task type. @@ -208,4 +211,5 @@ PUT _inference/text_embedding/alibabacloud_ai_search_embeddings } } ``` +% TEST[skip:TBD] diff --git a/explore-analyze/elastic-inference/inference-api/amazon-bedrock-inference-integration.md b/explore-analyze/elastic-inference/inference-api/amazon-bedrock-inference-integration.md index 79de7781d5..6b087deeb2 100644 --- a/explore-analyze/elastic-inference/inference-api/amazon-bedrock-inference-integration.md +++ b/explore-analyze/elastic-inference/inference-api/amazon-bedrock-inference-integration.md @@ -139,6 +139,7 @@ PUT _inference/text_embedding/amazon_bedrock_embeddings } } ``` +% TEST[skip:TBD] The next example shows how to create an {{infer}} endpoint called `amazon_bedrock_completion` to perform a `completion` task type. @@ -155,4 +156,5 @@ PUT _inference/completion/amazon_bedrock_completion } } ``` +% TEST[skip:TBD] diff --git a/explore-analyze/elastic-inference/inference-api/anthropic-inference-integration.md b/explore-analyze/elastic-inference/inference-api/anthropic-inference-integration.md index 32d5611105..dab0a6b296 100644 --- a/explore-analyze/elastic-inference/inference-api/anthropic-inference-integration.md +++ b/explore-analyze/elastic-inference/inference-api/anthropic-inference-integration.md @@ -129,4 +129,5 @@ PUT _inference/completion/anthropic_completion } } ``` +% TEST[skip:TBD] diff --git a/explore-analyze/elastic-inference/inference-api/azure-ai-studio-inference-integration.md b/explore-analyze/elastic-inference/inference-api/azure-ai-studio-inference-integration.md index 5bde120a6e..43e7187459 100644 --- a/explore-analyze/elastic-inference/inference-api/azure-ai-studio-inference-integration.md +++ b/explore-analyze/elastic-inference/inference-api/azure-ai-studio-inference-integration.md @@ -142,6 +142,7 @@ PUT _inference/text_embedding/azure_ai_studio_embeddings } } ``` +% TEST[skip:TBD] The next example shows how to create an {{infer}} endpoint called `azure_ai_studio_completion` to perform a `completion` task type. @@ -157,6 +158,7 @@ PUT _inference/completion/azure_ai_studio_completion } } ``` +% TEST[skip:TBD] The list of chat completion models that you can choose from in your deployment can be found in the [Azure AI Studio model explorer](https://ai.azure.com/explore/models?selectedTask=chat-completion). diff --git a/explore-analyze/elastic-inference/inference-api/azure-openai-inference-integration.md b/explore-analyze/elastic-inference/inference-api/azure-openai-inference-integration.md index b222927021..06b8aecc32 100644 --- a/explore-analyze/elastic-inference/inference-api/azure-openai-inference-integration.md +++ b/explore-analyze/elastic-inference/inference-api/azure-openai-inference-integration.md @@ -127,6 +127,7 @@ PUT _inference/text_embedding/azure_openai_embeddings } } ``` +% TEST[skip:TBD] The next example shows how to create an {{infer}} endpoint called `azure_openai_completion` to perform a `completion` task type. @@ -142,6 +143,7 @@ PUT _inference/completion/azure_openai_completion } } ``` +% TEST[skip:TBD] The list of chat completion models that you can choose from in your Azure OpenAI deployment can be found at the following places: diff --git a/explore-analyze/elastic-inference/inference-api/chat-completion-inference-api.md b/explore-analyze/elastic-inference/inference-api/chat-completion-inference-api.md index 1369efee44..c562fd95c1 100644 --- a/explore-analyze/elastic-inference/inference-api/chat-completion-inference-api.md +++ b/explore-analyze/elastic-inference/inference-api/chat-completion-inference-api.md @@ -408,6 +408,7 @@ POST _inference/chat_completion/openai-completion/_stream ] } ``` +% TEST[skip:TBD] The following example performs a chat completion using an Assistant message with `tool_calls`. @@ -437,6 +438,7 @@ POST _inference/chat_completion/openai-completion/_stream ] } ``` +% TEST[skip:TBD] 1. Each tool call needs a corresponding Tool message. 2. The corresponding Tool message. @@ -483,6 +485,7 @@ POST _inference/chat_completion/openai-completion/_stream } } ``` +% TEST[skip:TBD] The API returns the following response when a request is made to the OpenAI service: @@ -504,6 +507,7 @@ data: {"chat_completion":{"id":"chatcmpl-Ae0TWsy2VPnSfBbv5UztnSdYUMFP3","choices event: message data: [DONE] ``` +% NOTCONSOLE 1. The last object message of the stream contains the token usage information. diff --git a/explore-analyze/elastic-inference/inference-api/cohere-inference-integration.md b/explore-analyze/elastic-inference/inference-api/cohere-inference-integration.md index d654a5f9b9..4a03c76643 100644 --- a/explore-analyze/elastic-inference/inference-api/cohere-inference-integration.md +++ b/explore-analyze/elastic-inference/inference-api/cohere-inference-integration.md @@ -168,6 +168,7 @@ PUT _inference/text_embedding/cohere-embeddings } } ``` +% TEST[skip:TBD] The following example shows how to create an {{infer}} endpoint called `cohere-rerank` to perform a `rerank` task type. @@ -185,6 +186,7 @@ PUT _inference/rerank/cohere-rerank } } ``` +% TEST[skip:TBD] For more examples, also review the [Cohere documentation](https://docs.cohere.com/docs/elasticsearch-and-cohere#rerank-search-results-with-cohere-and-elasticsearch). diff --git a/explore-analyze/elastic-inference/inference-api/elastic-inference-service-eis.md b/explore-analyze/elastic-inference/inference-api/elastic-inference-service-eis.md index d6127e53f3..140d28d9d4 100644 --- a/explore-analyze/elastic-inference/inference-api/elastic-inference-service-eis.md +++ b/explore-analyze/elastic-inference/inference-api/elastic-inference-service-eis.md @@ -92,6 +92,7 @@ PUT _inference/sparse_embedding/elser-model-eis } } ``` +% TEST[skip:TBD] The following example shows how to create an {{infer}} endpoint called `chat-completion-endpoint` to perform a `chat_completion` task type. @@ -104,4 +105,5 @@ PUT /_inference/chat_completion/chat-completion-endpoint } } ``` +% TEST[skip:TBD] diff --git a/explore-analyze/elastic-inference/inference-api/elasticsearch-inference-integration.md b/explore-analyze/elastic-inference/inference-api/elasticsearch-inference-integration.md index f899600fed..49f584a0a0 100644 --- a/explore-analyze/elastic-inference/inference-api/elasticsearch-inference-integration.md +++ b/explore-analyze/elastic-inference/inference-api/elasticsearch-inference-integration.md @@ -128,6 +128,7 @@ PUT _inference/sparse_embedding/my-elser-model } } ``` +% TEST[skip:TBD] 1. Adaptive allocations will be enabled with the minimum of 1 and the maximum of 10 allocations. 2. The `model_id` must be the ID of one of the built-in ELSER models. Valid values are `.elser_model_2` and `.elser_model_2_linux-x86_64`. For further details, refer to the [ELSER model documentation](../../../explore-analyze/machine-learning/nlp/ml-nlp-elser.md). @@ -161,6 +162,7 @@ PUT _inference/rerank/my-elastic-rerank } } ``` +% TEST[skip:TBD] 1. The `model_id` must be the ID of the built-in Elastic Rerank model: `.rerank-v1`. 2. [Adaptive allocations](../../../deploy-manage/autoscaling/trained-model-autoscaling.md#enabling-autoscaling-through-apis-adaptive-allocations) will be enabled with the minimum of 1 and the maximum of 10 allocations. @@ -184,6 +186,7 @@ PUT _inference/text_embedding/my-e5-model } } ``` +% TEST[skip:TBD] 1. The `model_id` must be the ID of one of the built-in E5 models. Valid values are `.multilingual-e5-small` and `.multilingual-e5-small_linux-x86_64`. For further details, refer to the [E5 model documentation](../../../explore-analyze/machine-learning/nlp/ml-nlp-e5.md). @@ -210,6 +213,7 @@ PUT _inference/text_embedding/my-msmarco-minilm-model <1> } } ``` +% TEST[skip:TBD] 1. Provide an unique identifier for the inference endpoint. The `inference_id` must be unique and must not match the `model_id`. 2. The `model_id` must be the ID of a text embedding model which has already been [uploaded through Eland](../../../explore-analyze/machine-learning/nlp/ml-nlp-import-model.md#ml-nlp-import-script). @@ -237,6 +241,7 @@ PUT _inference/text_embedding/my-e5-model } } ``` +% TEST[skip:TBD] ## Using an existing model deployment with the `elasticsearch` service [inference-example-existing-deployment] @@ -252,6 +257,7 @@ PUT _inference/sparse_embedding/use_existing_deployment } } ``` +% TEST[skip:TBD] 1. The `deployment_id` of the already existing model deployment. @@ -276,4 +282,5 @@ The API response contains the `model_id`, and the threads and allocations settin } } ``` +% NOTCONSOLE diff --git a/explore-analyze/elastic-inference/inference-api/elser-inference-integration.md b/explore-analyze/elastic-inference/inference-api/elser-inference-integration.md index dac64a6b4d..cac9c3d98f 100644 --- a/explore-analyze/elastic-inference/inference-api/elser-inference-integration.md +++ b/explore-analyze/elastic-inference/inference-api/elser-inference-integration.md @@ -124,6 +124,7 @@ PUT _inference/sparse_embedding/my-elser-model } } ``` +% TEST[skip:TBD] ## ELSER service example without adaptive allocations [inference-example-elser] @@ -147,6 +148,7 @@ PUT _inference/sparse_embedding/my-elser-model } } ``` +% TEST[skip:TBD] Example response: @@ -162,6 +164,7 @@ Example response: "task_settings": {} } ``` +% NOTCONSOLE ::::{note} You might see a 502 bad gateway error in the response when using the {{kib}} Console. This error usually just reflects a timeout, while the model downloads in the background. You can check the download progress in the {{ml-app}} UI. If using the Python client, you can set the `timeout` parameter to a higher value. diff --git a/explore-analyze/elastic-inference/inference-api/google-ai-studio-inference-integration.md b/explore-analyze/elastic-inference/inference-api/google-ai-studio-inference-integration.md index 1ee88b1c7f..db7a0c3d18 100644 --- a/explore-analyze/elastic-inference/inference-api/google-ai-studio-inference-integration.md +++ b/explore-analyze/elastic-inference/inference-api/google-ai-studio-inference-integration.md @@ -93,4 +93,5 @@ PUT _inference/completion/google_ai_studio_completion } } ``` +% TEST[skip:TBD] diff --git a/explore-analyze/elastic-inference/inference-api/google-vertex-ai-inference-integration.md b/explore-analyze/elastic-inference/inference-api/google-vertex-ai-inference-integration.md index afcf2809d8..8e16f0bca4 100644 --- a/explore-analyze/elastic-inference/inference-api/google-vertex-ai-inference-integration.md +++ b/explore-analyze/elastic-inference/inference-api/google-vertex-ai-inference-integration.md @@ -120,6 +120,7 @@ PUT _inference/text_embedding/google_vertex_ai_embeddings } } ``` +% TEST[skip:TBD] The next example shows how to create an {{infer}} endpoint called `google_vertex_ai_rerank` to perform a `rerank` task type. @@ -133,4 +134,5 @@ PUT _inference/rerank/google_vertex_ai_rerank } } ``` +% TEST[skip:TBD] diff --git a/explore-analyze/elastic-inference/inference-api/huggingface-inference-integration.md b/explore-analyze/elastic-inference/inference-api/huggingface-inference-integration.md index ced0ccd53f..6dccefba2b 100644 --- a/explore-analyze/elastic-inference/inference-api/huggingface-inference-integration.md +++ b/explore-analyze/elastic-inference/inference-api/huggingface-inference-integration.md @@ -97,6 +97,7 @@ PUT _inference/text_embedding/hugging-face-embeddings } } ``` +% TEST[skip:TBD] 1. A valid Hugging Face access token. You can find on the [settings page of your account](https://huggingface.co/settings/tokens). 2. The {{infer}} endpoint URL you created on Hugging Face. diff --git a/explore-analyze/elastic-inference/inference-api/jinaai-inference-integration.md b/explore-analyze/elastic-inference/inference-api/jinaai-inference-integration.md index 40ee6e4715..b069d7eebd 100644 --- a/explore-analyze/elastic-inference/inference-api/jinaai-inference-integration.md +++ b/explore-analyze/elastic-inference/inference-api/jinaai-inference-integration.md @@ -141,6 +141,7 @@ PUT _inference/text_embedding/jinaai-embeddings } } ``` +% TEST[skip:uses ML] Then, we create the `rerank` service: @@ -158,6 +159,7 @@ PUT _inference/rerank/jinaai-rerank } } ``` +% TEST[skip:uses ML] Now we can create an index that will use `jinaai-embeddings` service to index the documents. @@ -174,6 +176,7 @@ PUT jinaai-index } } ``` +% TEST[skip:uses ML] ```console PUT jinaai-index/_bulk @@ -184,6 +187,7 @@ PUT jinaai-index/_bulk { "index" : { "_index" : "jinaai-index", "_id" : "3" } } {"content": "Her dedication to preserving these delicate underwater environments has inspired a new generation of conservationists."} ``` +% TEST[skip:uses ML] Now, with the index created, we can search with and without the reranker service. @@ -198,6 +202,7 @@ GET jinaai-index/_search } } ``` +% TEST[skip:uses ML] ```console POST jinaai-index/_search @@ -222,4 +227,5 @@ POST jinaai-index/_search } } ``` +% TEST[skip:uses ML] diff --git a/explore-analyze/elastic-inference/inference-api/mistral-inference-integration.md b/explore-analyze/elastic-inference/inference-api/mistral-inference-integration.md index fe374bb178..fa71c2d2e1 100644 --- a/explore-analyze/elastic-inference/inference-api/mistral-inference-integration.md +++ b/explore-analyze/elastic-inference/inference-api/mistral-inference-integration.md @@ -100,6 +100,7 @@ PUT _inference/text_embedding/mistral-embeddings-test } } ``` +% TEST[skip:TBD] 1. The `model` must be the ID of a text embedding model which can be found in the [Mistral models documentation](https://docs.mistral.ai/getting-started/models/). diff --git a/explore-analyze/elastic-inference/inference-api/openai-inference-integration.md b/explore-analyze/elastic-inference/inference-api/openai-inference-integration.md index 712922678d..632be61c0c 100644 --- a/explore-analyze/elastic-inference/inference-api/openai-inference-integration.md +++ b/explore-analyze/elastic-inference/inference-api/openai-inference-integration.md @@ -136,6 +136,7 @@ PUT _inference/text_embedding/openai-embeddings } } ``` +% TEST[skip:TBD] The next example shows how to create an {{infer}} endpoint called `openai-completion` to perform a `completion` task type. @@ -149,3 +150,4 @@ PUT _inference/completion/openai-completion } } ``` +% TEST[skip:TBD] diff --git a/explore-analyze/machine-learning/nlp/nlp-end-to-end-tutorial.md b/explore-analyze/machine-learning/nlp/nlp-end-to-end-tutorial.md index a8f9f9bbab..03e6e754cf 100644 --- a/explore-analyze/machine-learning/nlp/nlp-end-to-end-tutorial.md +++ b/explore-analyze/machine-learning/nlp/nlp-end-to-end-tutorial.md @@ -97,6 +97,7 @@ Let’s say our photo comments look like this when they are uploaded as a docume ... } ``` +% NOTCONSOLE We want to run our documents through an inference processor that uses the trained model we uploaded to determine if the comments are positive. To do this, we first need to set up an Elasticsearch index. @@ -116,6 +117,7 @@ PUT search-photo-comments/_mapping } } ``` +% NOTCONSOLE Now it’s time to create an inference pipeline. @@ -144,12 +146,14 @@ POST search-photo-comments/_doc/my-new-doc?pipeline=search-photo-comments "_run_ml_inference": true } ``` +% NOTCONSOLE Once the document is indexed, use the API to retrieve it and view the enriched data. ```js GET search-photo-comments/_doc/my-new-doc ``` +% NOTCONSOLE ```js { @@ -172,6 +176,7 @@ GET search-photo-comments/_doc/my-new-doc } } ``` +% NOTCONSOLE The document has new fields with the enriched data. The `ml.inference.positivity_result` field is an object with the analysis from the machine learning model. The model we used predicted with 99.98% confidence that the analyzed text is positive. diff --git a/explore-analyze/query-filter/aggregations.md b/explore-analyze/query-filter/aggregations.md index 4789c0b0c5..f1ab09173f 100644 --- a/explore-analyze/query-filter/aggregations.md +++ b/explore-analyze/query-filter/aggregations.md @@ -37,6 +37,8 @@ GET /my-index-000001/_search } } ``` +% TEST[setup:my_index] +% TEST[s/my-field/http.request.method/] Aggregation results are in the response’s `aggregations` object: @@ -67,6 +69,10 @@ Aggregation results are in the response’s `aggregations` object: } } ``` +% TESTRESPONSE[s/"took": 78/"took": "$body.took"/] +% TESTRESPONSE[s/...$/"took": "$body.took", "timed_out": false, "_shards": "$body._shards", /] +% TESTRESPONSE[s/"hits": [...]/"hits": "$body.hits.hits"/] +% TESTRESPONSE[s/"buckets": []/"buckets":[{"key":"get","doc_count":5}]/] 1. Results for the `my-agg-name` aggregation. @@ -94,6 +100,8 @@ GET /my-index-000001/_search } } ``` +% TEST[setup:my_index] +% TEST[s/my-field/http.request.method/] ## Return only aggregation results [return-only-agg-results] @@ -112,6 +120,8 @@ GET /my-index-000001/_search } } ``` +% TEST[setup:my_index] +% TEST[s/my-field/http.request.method/] ## Run multiple aggregations [run-multiple-aggs] @@ -134,6 +144,9 @@ GET /my-index-000001/_search } } ``` +% TEST[setup:my_index] +% TEST[s/my-field/http.request.method/] +% TEST[s/my-other-field/http.response.bytes/] ## Run sub-aggregations [run-sub-aggs] @@ -158,6 +171,10 @@ GET /my-index-000001/_search } } ``` +% TEST[setup:my_index] +% TEST[s/_search/_search?size=0/] +% TEST[s/my-field/http.request.method/] +% TEST[s/my-other-field/http.response.bytes/] The response nests sub-aggregation results under their parent aggregation: @@ -181,6 +198,9 @@ The response nests sub-aggregation results under their parent aggregation: } } ``` +% TESTRESPONSE[s/.../"took": "$body.took", "timed_out": false, "_shards": "$body._shards", "hits": "$body.hits",/] +% TESTRESPONSE[s/"key": "foo"/"key": "get"/] +% TESTRESPONSE[s/"value": 75.0/"value": $body.aggregations.my-agg-name.buckets.0.my-sub-agg-name.value/] 1. Results for the parent aggregation, `my-agg-name`. 2. Results for `my-agg-name`'s sub-aggregation, `my-sub-agg-name`. @@ -204,6 +224,8 @@ GET /my-index-000001/_search } } ``` +% TEST[setup:my_index] +% TEST[s/_search/_search?size=0/] The response returns the `meta` object in place: @@ -222,6 +244,7 @@ The response returns the `meta` object in place: } } ``` +% TESTRESPONSE[s/.../"took": "$body.took", "timed_out": false, "_shards": "$body._shards", "hits": "$body.hits",/] ## Return the aggregation type [return-agg-type] @@ -240,6 +263,9 @@ GET /my-index-000001/_search?typed_keys } } ``` +% TEST[setup:my_index] +% TEST[s/typed_keys/typed_keys&size=0/] +% TEST[s/my-field/http.response.bytes/] The response returns the aggregation type as a prefix to the aggregation’s name. @@ -257,6 +283,8 @@ Some aggregations return a different aggregation type from the type in the reque } } ``` +% TESTRESPONSE[s/.../"took": "$body.took", "timed_out": false, "_shards": "$body._shards", "hits": "$body.hits",/] +% TESTRESPONSE[s/"buckets": []/"buckets":[{"key":1070000.0,"doc_count":5}]/] 1. The aggregation type, `histogram`, followed by a `#` separator and the aggregation’s name, `my-agg-name`. @@ -283,6 +311,7 @@ GET /my-index-000001/_search?size=0 } } ``` +% TEST[setup:my_index] Scripts calculate field values dynamically, which adds a little overhead to the aggregation. In addition to the time spent calculating, some aggregations like [`terms`](elasticsearch://reference/data-analysis/aggregations/search-aggregations-bucket-terms-aggregation.md) and [`filters`](elasticsearch://reference/data-analysis/aggregations/search-aggregations-bucket-filters-aggregation.md) can’t use some of their optimizations with runtime fields. In total, performance costs for using a runtime field varies from aggregation to aggregation. diff --git a/explore-analyze/query-filter/aggregations/tutorial-analyze-ecommerce-data-with-aggregations-using-query-dsl.md b/explore-analyze/query-filter/aggregations/tutorial-analyze-ecommerce-data-with-aggregations-using-query-dsl.md index def3d4f772..bb77e1c249 100644 --- a/explore-analyze/query-filter/aggregations/tutorial-analyze-ecommerce-data-with-aggregations-using-query-dsl.md +++ b/explore-analyze/query-filter/aggregations/tutorial-analyze-ecommerce-data-with-aggregations-using-query-dsl.md @@ -45,6 +45,7 @@ Before we start analyzing the data, let’s examine the structure of the documen ```console GET kibana_sample_data_ecommerce/_mapping ``` +% TEST[skip:Using Kibana sample data] The response shows the field mappings for the `kibana_sample_data_ecommerce` index. @@ -305,6 +306,7 @@ GET kibana_sample_data_ecommerce/_search } } ``` +% TEST[skip:Using Kibana sample data] 1. Set `size` to 0 to avoid returning matched documents in the response and return only the aggregation results 2. A meaningful name that describes what this metric represents @@ -337,6 +339,7 @@ GET kibana_sample_data_ecommerce/_search } } ``` +% TEST[skip:Using Kibana sample data] 1. Total number of orders in the dataset 2. `hits` is empty because we set `size` to 0 @@ -362,6 +365,7 @@ GET kibana_sample_data_ecommerce/_search } } ``` +% TEST[skip:Using Kibana sample data] 1. A descriptive name for this set of statistics 2. `stats` returns count, min, max, avg, and sum at once @@ -381,6 +385,7 @@ GET kibana_sample_data_ecommerce/_search } } ``` +% TEST[skip:Using Kibana sample data] 1. `"count"`: Total number of orders in the dataset 2. `"min"`: Lowest individual order value in the dataset @@ -418,6 +423,7 @@ GET kibana_sample_data_ecommerce/_search } } ``` +% TEST[skip:Using Kibana sample data] 1. Name reflecting the business purpose of this breakdown 2. `terms` aggregation groups documents by field values @@ -475,6 +481,7 @@ GET kibana_sample_data_ecommerce/_search } } ``` +% TEST[skip:Using Kibana sample data] 1. Due to Elasticsearch’s distributed architecture, when [terms aggregations](elasticsearch://reference/data-analysis/aggregations/search-aggregations-bucket-terms-aggregation.md) run across multiple shards, the doc counts may have a small margin of error. This value indicates the maximum possible error in the counts. 2. Count of documents in categories beyond the requested size. @@ -504,6 +511,7 @@ GET kibana_sample_data_ecommerce/_search } } ``` +% TEST[skip:Using Kibana sample data] 1. Descriptive name for the time-series aggregation results. 2. The `date_histogram` aggregation groups documents into time-based buckets, similar to terms aggregation but for dates. @@ -694,6 +702,7 @@ GET kibana_sample_data_ecommerce/_search } } ``` +% TEST[skip:Using Kibana sample data] 1. Results of our named aggregation "daily_orders" 2. Time-based buckets from date_histogram aggregation @@ -743,6 +752,7 @@ GET kibana_sample_data_ecommerce/_search } } ``` +% TEST[skip:Using Kibana sample data] 1. Order categories by their total revenue instead of count 2. Define metrics to calculate within each category @@ -780,6 +790,7 @@ GET kibana_sample_data_ecommerce/_search } } ``` +% TEST[skip:Using Kibana sample data] 1. Category name 2. Number of orders @@ -825,6 +836,7 @@ GET kibana_sample_data_ecommerce/_search } } ``` +% TEST[skip:Using Kibana sample data] 1. Daily revenue 2. Uses the [`cardinality`](elasticsearch://reference/data-analysis/aggregations/search-aggregations-metrics-cardinality-aggregation.md) aggregation to count unique customers per day @@ -1331,6 +1343,7 @@ GET kibana_sample_data_ecommerce/_search } } ``` +% TEST[skip:Using Kibana sample data] 1. Calculate daily revenue first. 2. Create a smoothed version of the daily revenue. @@ -1708,6 +1721,7 @@ GET kibana_sample_data_ecommerce/_search } } ``` +% TEST[skip:Using Kibana sample data] 1. Date of the bucket is in default ISO format because we didn’t specify a format 2. Number of orders for this day @@ -1752,6 +1766,7 @@ GET kibana_sample_data_ecommerce/_search } } ``` +% TEST[skip:Using Kibana sample data] 1. Name for our running total 2. `cumulative_sum` adds up values across buckets @@ -2126,6 +2141,7 @@ GET kibana_sample_data_ecommerce/_search } } ``` +% TEST[skip:Using Kibana sample data] 1. `daily_sales`: Results from our daily sales date histogram 2. `buckets`: Array of time-based buckets diff --git a/explore-analyze/query-filter/languages/eql.md b/explore-analyze/query-filter/languages/eql.md index 42646e5c62..9ba6ba1a7c 100644 --- a/explore-analyze/query-filter/languages/eql.md +++ b/explore-analyze/query-filter/languages/eql.md @@ -48,6 +48,7 @@ GET /my-data-stream/_eql/search """ } ``` +% TEST[setup:sec_logs] By default, basic EQL queries return the 10 most recent matching events in the `hits.events` property. These hits are sorted by timestamp, converted to milliseconds since the [Unix epoch](https://en.wikipedia.org/wiki/Unix_time), in ascending order. @@ -103,6 +104,10 @@ By default, basic EQL queries return the 10 most recent matching events in the ` } } ``` +% TESTRESPONSE[s/"took": 60/"took": $body.took/] +% TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.12.07-000001"/"_index": $body.hits.events.0._index/] +% TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.events.0._id/] +% TESTRESPONSE[s/"_id": "xLkCaj4EujzdNSxfYLbO"/"_id": $body.hits.events.1._id/] Use the `size` parameter to get a smaller or larger set of hits: @@ -115,6 +120,7 @@ GET /my-data-stream/_eql/search "size": 50 } ``` +% TEST[setup:sec_logs] ## Search for a sequence of events [eql-search-sequence] @@ -131,6 +137,7 @@ GET /my-data-stream/_eql/search """ } ``` +% TEST[setup:sec_logs] The response’s `hits.sequences` property contains the 10 most recent matching sequences. @@ -187,6 +194,11 @@ The response’s `hits.sequences` property contains the 10 most recent matching } } ``` +% TESTRESPONSE[s/ ...\n/"is_partial": false, "is_running": false, "took": $body.took, "timed_out": false,/] +% TESTRESPONSE[s/"total": ...,/"total": { "value": 1, "relation": "eq" },/] +% TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.12.07-000001"/"_index": $body.hits.sequences.0.events.0._index/] +% TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.sequences.0.events.0._id/] +% TESTRESPONSE[s/"_id": "yDwnGIJouOYGBzP0ZE9n"/"_id": $body.hits.sequences.0.events.1._id/] Use [`with maxspan`](elasticsearch://reference/query-languages/eql-syntax.md#eql-with-maxspan-keywords) to constrain matching sequences to a timespan: @@ -200,6 +212,7 @@ GET /my-data-stream/_eql/search """ } ``` +% TEST[setup:sec_logs] Use `!` to match [missing events](elasticsearch://reference/query-languages/eql-syntax.md#eql-missing-events): events in a sequence that do not meet a condition within a given timespan: @@ -214,6 +227,7 @@ GET /my-data-stream/_eql/search """ } ``` +% TEST[setup:sec_logs] Missing events are indicated in the response as `missing": true`: @@ -275,6 +289,11 @@ Missing events are indicated in the response as `missing": true`: } } ``` +% TESTRESPONSE[s/ ...\n/"is_partial": false, "is_running": false, "took": $body.took, "timed_out": false,/] +% TESTRESPONSE[s/"total": ...,/"total": { "value": 1, "relation": "eq" },/] +% TESTRESPONSE[s/"_index": ".ds-my-data-stream-2023.07.04-000001"/"_index": $body.hits.sequences.0.events.0._index/] +% TESTRESPONSE[s/"_id": "AnpTIYkBrVQ2QEgsWg94"/"_id": $body.hits.sequences.0.events.0._id/] +% TESTRESPONSE[s/"_id": "BHpTIYkBrVQ2QEgsWg94"/"_id": $body.hits.sequences.0.events.2._id/] Use the [`by` keyword](elasticsearch://reference/query-languages/eql-syntax.md#eql-by-keyword) to match events that share the same field values: @@ -288,6 +307,7 @@ GET /my-data-stream/_eql/search """ } ``` +% TEST[setup:sec_logs] If a field value should be shared across all events, use the `sequence by` keyword. The following query is equivalent to the previous one. @@ -301,6 +321,7 @@ GET /my-data-stream/_eql/search """ } ``` +% TEST[setup:sec_logs] The `hits.sequences.join_keys` property contains the shared field values. @@ -319,6 +340,9 @@ The `hits.sequences.join_keys` property contains the shared field values. } } ``` +% TESTRESPONSE[s/ ...\n/"is_partial": false, "is_running": false, "took": $body.took, "timed_out": false,/] +% TESTRESPONSE[s/"hits": ...,/"hits": { "total": { "value": 1, "relation": "eq" },/] +% TESTRESPONSE[s/"events": .../"events": $body.hits.sequences.0.events/] Use the [`until` keyword](elasticsearch://reference/query-languages/eql-syntax.md#eql-until-keyword) to specify an expiration event for sequences. Matching sequences must end before this event. @@ -333,6 +357,7 @@ GET /my-data-stream/_eql/search """ } ``` +% TEST[setup:sec_logs] ## Sample chronologically unordered events [eql-search-sample] @@ -566,6 +591,7 @@ GET /my-index*/_eql/search """ } ``` +% TEST[continued] By default, the response’s `hits.sequences` property contains up to 10 samples. Each sample has a set of `join_keys` and an array with one matching event for each of the filters. Events are returned in the order of the filters they match: @@ -692,6 +718,7 @@ By default, the response’s `hits.sequences` property contains up to 10 samples } } ``` +% TESTRESPONSE[skip:Response is illustrative only] 1. The events in the first sample have a value of `doom` for `host`. 2. This event matches the first filter. @@ -713,6 +740,7 @@ GET /my-index*/_eql/search """ } ``` +% TEST[continued] This query will return samples where each of the events shares the same value for `os` or `op_sys`, as well as for `host`. For example: @@ -842,6 +870,7 @@ This query will return samples where each of the events shares the same value fo } } ``` +% TESTRESPONSE[skip:Response is illustrative only] 1. The events in this sample have a value of `doom` for `host` and a value of `redhat` for `os` or `op_sys`. @@ -861,6 +890,7 @@ GET /my-index*/_eql/search """ } ``` +% TEST[continued] 1. Retrieve up to 2 samples per set of join keys. 2. Retrieve up to 20 samples in total. @@ -881,6 +911,7 @@ GET /my-data-stream/_eql/search?filter_path=hits.events._source.@timestamp,hits. """ } ``` +% TEST[setup:sec_logs] The API returns the following response. @@ -938,6 +969,7 @@ GET /my-data-stream/_eql/search?filter_path=-hits.events._source ] } ``` +% TEST[setup:sec_logs] 1. Both full field names and wildcard patterns are accepted. 2. Use the `format` parameter to apply a custom format for the field’s values. @@ -986,6 +1018,11 @@ The response includes values as a flat list in the `fields` section for each hit } } ``` +% TESTRESPONSE[s/ ...\n/"is_partial": false, "is_running": false, "took": $body.took, "timed_out": false,/] +% TESTRESPONSE[s/"total": ...,/"total": { "value": 2, "relation": "eq" },/] +% TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.12.07-000001"/"_index": $body.hits.events.0._index/] +% TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.events.0._id/] +% TESTRESPONSE[s/ ....\n/$body.hits.events.1/] ## Use runtime fields [eql-use-runtime-fields] @@ -1012,6 +1049,7 @@ GET /my-data-stream/_eql/search?filter_path=-hits.events._source ] } ``` +% TEST[setup:sec_logs] The API returns: @@ -1038,6 +1076,11 @@ The API returns: } } ``` +% TESTRESPONSE[s/ ...\n/"is_partial": false, "is_running": false, "took": $body.took, "timed_out": false,/] +% TESTRESPONSE[s/"total": ...,/"total": { "value": 2, "relation": "eq" },/] +% TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.12.07-000001"/"_index": $body.hits.events.0._index/] +% TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.events.0._id/] +% TESTRESPONSE[s/ ....\n/$body.hits.events.1/] ## Specify a timestamp or event category field [specify-a-timestamp-or-event-category-field] @@ -1054,6 +1097,7 @@ GET /my-data-stream/_eql/search """ } ``` +% TEST[setup:sec_logs] The event category field must be mapped as a [`keyword`](elasticsearch://reference/elasticsearch/mapping-reference/keyword.md) family field type. The timestamp field should be mapped as a [`date`](elasticsearch://reference/elasticsearch/mapping-reference/date.md) field type. [`date_nanos`](elasticsearch://reference/elasticsearch/mapping-reference/date_nanos.md) timestamp fields are not supported. You cannot use a [`nested`](elasticsearch://reference/elasticsearch/mapping-reference/nested.md) field or the sub-fields of a `nested` field as the timestamp or event category field. @@ -1075,6 +1119,7 @@ GET /my-data-stream/_eql/search """ } ``` +% TEST[setup:sec_logs] ## Filter using Query DSL [eql-search-filter-query-dsl] @@ -1097,6 +1142,7 @@ GET /my-data-stream/_eql/search """ } ``` +% TEST[setup:sec_logs] ## Run an async EQL search [eql-search-async] @@ -1114,6 +1160,7 @@ GET /my-data-stream/_eql/search """ } ``` +% TEST[setup:sec_logs] If the request doesn’t finish within the timeout period, the search becomes async and returns a response that includes: @@ -1133,12 +1180,18 @@ The async search continues to run in the background without blocking other reque "hits": ... } ``` +% TESTRESPONSE[s/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=/$body.id/] +% TESTRESPONSE[s/"is_partial": true/"is_partial": $body.is_partial/] +% TESTRESPONSE[s/"is_running": true/"is_running": $body.is_running/] +% TESTRESPONSE[s/"took": 2000/"took": $body.took/] +% TESTRESPONSE[s/"hits": .../"hits": $body.hits/] To check the progress of an async search, use the [get async EQL search API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-eql-get) with the search ID. Specify how long you’d like for complete results in the `wait_for_completion_timeout` parameter. ```console GET /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?wait_for_completion_timeout=2s ``` +% TEST[skip: no access to search ID] If the response’s `is_running` value is `false`, the async search has finished. If the `is_partial` value is `false`, the returned search results are complete. @@ -1152,12 +1205,16 @@ If the response’s `is_running` value is `false`, the async search has finished "hits": ... } ``` +% TESTRESPONSE[s/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=/$body.id/] +% TESTRESPONSE[s/"took": 2000/"took": $body.took/] +% TESTRESPONSE[s/"hits": .../"hits": $body.hits/] Another more lightweight way to check the progress of an async search is to use the [get async EQL status API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-eql-get-status) with the search ID. ```console GET /_eql/search/status/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE= ``` +% TEST[skip: no access to search ID] ```console-result { @@ -1168,6 +1225,8 @@ GET /_eql/search/status/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FS "completion_status": 200 } ``` +% TESTRESPONSE[s/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=/$body.id/] +% TESTRESPONSE[s/"expiration_time_in_millis": 1611690295000/"expiration_time_in_millis": $body.expiration_time_in_millis/] ## Change the search retention period [eql-search-store-async-eql-search] @@ -1184,18 +1243,21 @@ GET /my-data-stream/_eql/search """ } ``` +% TEST[setup:sec_logs] You can use the [get async EQL search API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-eql-get)'s `keep_alive` parameter to later change the retention period. The new retention period starts after the get request runs. ```console GET /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?keep_alive=5d ``` +% TEST[skip: no access to search ID] Use the [delete async EQL search API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-eql-delete) to manually delete an async EQL search before the `keep_alive` period ends. If the search is still ongoing, {{es}} cancels the search request. ```console DELETE /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE= ``` +% TEST[skip: no access to search ID] ## Store synchronous EQL searches [eql-search-store-sync-eql-search] @@ -1212,6 +1274,7 @@ GET /my-data-stream/_eql/search """ } ``` +% TEST[setup:sec_logs] The response includes a search ID. `is_partial` and `is_running` are `false`, indicating the EQL search was synchronous and returned complete results. @@ -1225,12 +1288,16 @@ The response includes a search ID. `is_partial` and `is_running` are `false`, in "hits": ... } ``` +% TESTRESPONSE[s/FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=/$body.id/] +% TESTRESPONSE[s/"took": 52/"took": $body.took/] +% TESTRESPONSE[s/"hits": .../"hits": $body.hits/] Use the [get async EQL search API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-eql-get) to get the same results later: ```console GET /_eql/search/FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY= ``` +% TEST[skip: no access to search ID] Saved synchronous searches are still subject to the `keep_alive` parameter’s retention period. When this period ends, the search and its results are deleted. @@ -1271,6 +1338,8 @@ PUT /_cluster/settings } } ``` +% TEST[setup:host] +% TEST[s/127.0.0.1:930\d+/${transport_host}/] To target a data stream or index on a remote cluster, use the `:` syntax. @@ -1282,6 +1351,9 @@ GET /cluster_one:my-data-stream,cluster_two:my-data-stream/_eql/search """ } ``` +% TEST[continued] +% TEST[setup:sec_logs] +% TEST[teardown:data_stream_cleanup] ## EQL circuit breaker settings [eql-circuit-breaker] diff --git a/explore-analyze/query-filter/languages/esql-cross-clusters.md b/explore-analyze/query-filter/languages/esql-cross-clusters.md index 4023c3ef3b..4231e3dd3c 100644 --- a/explore-analyze/query-filter/languages/esql-cross-clusters.md +++ b/explore-analyze/query-filter/languages/esql-cross-clusters.md @@ -168,6 +168,8 @@ PUT _cluster/settings } } ``` +% TEST[setup:host] +% TEST[s/35.238.149.\d+:930\d+/${transport_host}/] 1. Since `skip_unavailable` was not set on `cluster_three`, it uses the default of `false`. See the [Optional remote clusters](#ccq-skip-unavailable-clusters) section for details. @@ -215,6 +217,8 @@ POST /_query/async?format=json "include_ccs_metadata": true } ``` +% TEST[setup:my_index] +% TEST[s/cluster_one:my-index-000001,cluster_two:my-index//] Which returns: @@ -282,6 +286,7 @@ Which returns: } } ``` +% TEST[skip: cross-cluster testing env not set up] 1. How long the entire search (across all clusters) took, in milliseconds. 2. This section of counters shows all possible cluster search states and how many cluster searches are currently in that state. The clusters can have one of the following statuses: **running**, **successful** (searches on all shards were successful), **skipped** (the search failed on a cluster marked with `skip_unavailable`=`true`), **failed** (the search failed on a cluster marked with `skip_unavailable`=`false`) or **partial** (the search was [interrupted](https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-async-query-stop-api.html) before finishing). @@ -305,6 +310,8 @@ POST /_query/async?format=json "include_ccs_metadata": true } ``` +% TEST[continued] +% TEST[s/cluster_one:my-index*,cluster_two:logs*/my-index-000001/] Which returns: diff --git a/explore-analyze/query-filter/languages/esql-rest.md b/explore-analyze/query-filter/languages/esql-rest.md index f8b2ecbe75..3b9b6738b5 100644 --- a/explore-analyze/query-filter/languages/esql-rest.md +++ b/explore-analyze/query-filter/languages/esql-rest.md @@ -23,6 +23,7 @@ POST /_query?format=txt "query": "FROM library | KEEP author, name, page_count, release_date | SORT page_count DESC | LIMIT 5" } ``` +% TEST[setup:library] Which returns: @@ -35,6 +36,8 @@ Frank Herbert |Dune |604 |1965-06-01T00:00:00.000Z Alastair Reynolds|Revelation Space |585 |2000-03-15T00:00:00.000Z James S.A. Corey |Leviathan Wakes |561 |2011-06-02T00:00:00.000Z ``` +% TESTRESPONSE[s/|/\|/ s/+/\+/] +% TESTRESPONSE[non_json] ### Kibana Console [esql-kibana-console] @@ -52,6 +55,7 @@ POST /_query?format=txt """ } ``` +% TEST[setup:library] ### Response formats [esql-rest-format] @@ -103,6 +107,7 @@ POST /_query?format=txt } } ``` +% TEST[setup:library] Which returns: @@ -111,6 +116,8 @@ Which returns: ---------------+------------------------------------+---------------+------------------------ Douglas Adams |The Hitchhiker's Guide to the Galaxy|180 |1979-10-12T00:00:00.000Z ``` +% TESTRESPONSE[s/|/\|/ s/+/\+/] +% TESTRESPONSE[non_json] ### Columnar results [esql-rest-columnar] @@ -129,6 +136,7 @@ POST /_query?format=json "columnar": true } ``` +% TEST[setup:library] Which returns: @@ -150,6 +158,7 @@ Which returns: ] } ``` +% TESTRESPONSE[s/"took": 28/"took": "$body.took"/] ### Returning localized results [esql-locale-param] @@ -172,6 +181,7 @@ POST /_query """ } ``` +% TEST[setup:library] ### Passing parameters to a query [esql-rest-params] @@ -191,6 +201,7 @@ POST /_query """ } ``` +% TEST[setup:library] To avoid any attempts of hacking or code injection, extract the values in a separate list of parameters. Use question mark placeholders (`?`) in the query string for each of the parameters: @@ -208,6 +219,7 @@ POST /_query "params": [300, "Frank Herbert", 0] } ``` +% TEST[setup:library] The parameters can be named parameters or positional parameters. @@ -227,6 +239,7 @@ POST /_query "params": [{"page_count" : 300}, {"author" : "Frank Herbert"}, {"count" : 0}] } ``` +% TEST[setup:library] Positional parameters use question mark placeholders (`?`) followed by an integer. @@ -244,6 +257,7 @@ POST /_query "params": [300, "Frank Herbert", 0] } ``` +% TEST[setup:library] ### Running an async {{esql}} query [esql-rest-async-query] @@ -267,6 +281,8 @@ POST /_query/async "wait_for_completion_timeout": "2s" } ``` +% TEST[setup:library] +% TEST[skip:awaitsfix https://github.com/elastic/elasticsearch/issues/104013] If the results are not available within the given timeout period, 2 seconds in this case, no results are returned but rather a response that includes: @@ -281,12 +297,14 @@ The query continues to run in the background without blocking other requests. "is_running": true } ``` +% TEST[skip: no access to query ID - may return response values] To check the progress of an async query, use the [{{esql}} async query get API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-esql-async-query-get) with the query ID. Specify how long you’d like to wait for complete results in the `wait_for_completion_timeout` parameter. ```console GET /_query/async/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?wait_for_completion_timeout=30s ``` +% TEST[skip: no access to query ID - may return response values] If the response’s `is_running` value is `false`, the query has finished and the results are returned, along with the `took` time for the query. @@ -297,9 +315,11 @@ If the response’s `is_running` value is `false`, the query has finished and th "columns": ... } ``` +% TEST[skip: no access to query ID - may return response values] Use the [{{esql}} async query delete API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-esql-async-query-delete) to delete an async query before the `keep_alive` period ends. If the query is still running, {{es}} cancels it. ```console DELETE /_query/async/FmdMX2pIang3UWhLRU5QS0lqdlppYncaMUpYQ05oSkpTc3kwZ21EdC1tbFJXQToxOTI= ``` +% TEST[skip: no access to query ID] diff --git a/explore-analyze/query-filter/languages/esql-task-management.md b/explore-analyze/query-filter/languages/esql-task-management.md index b8593f13ef..c34fd12e63 100644 --- a/explore-analyze/query-filter/languages/esql-task-management.md +++ b/explore-analyze/query-filter/languages/esql-task-management.md @@ -38,6 +38,7 @@ Which returns a list of statuses like this: "headers" : { } } ``` +% NOTCONSOLE 1. The user submitted query. 2. Time the query has been running. diff --git a/explore-analyze/query-filter/languages/example-detect-threats-with-eql.md b/explore-analyze/query-filter/languages/example-detect-threats-with-eql.md index 902cb08fb8..55be66ab82 100644 --- a/explore-analyze/query-filter/languages/example-detect-threats-with-eql.md +++ b/explore-analyze/query-filter/languages/example-detect-threats-with-eql.md @@ -71,6 +71,7 @@ GET /my-data-stream/_eql/search?filter_path=-hits.events <1> "size": 200 <3> } ``` +% TEST[setup:atomic_red_regsvr32] 1. `?filter_path=-hits.events` excludes the `hits.events` property from the response. This search is only intended to get an event count, not a list of matching events. 2. Matches any event with a `process.name` of `regsvr32.exe`. @@ -93,6 +94,7 @@ The response returns 143 related events. } } ``` +% TESTRESPONSE[s/"took": 60/"took": $body.took/] ## Check for command line artifacts [eql-ex-check-for-command-line-artifacts] @@ -107,6 +109,7 @@ GET /my-data-stream/_eql/search """ } ``` +% TEST[setup:atomic_red_regsvr32] The query matches one event with an `event.type` of `creation`, indicating the start of a `regsvr32.exe` process. Based on the event’s `process.command_line` value, `regsvr32.exe` used `scrobj.dll` to register a script, `RegSvr32.sct`. This fits the behavior of a Squiblydoo attack. @@ -153,6 +156,9 @@ The query matches one event with an `event.type` of `creation`, indicating the s } } ``` +% TESTRESPONSE[s/ ...\n/"is_partial": false, "is_running": false, "took": $body.took, "timed_out": false,/] +% TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.12.07-000001"/"_index": $body.hits.events.0._index/] +% TESTRESPONSE[s/"_id": "gl5MJXMBMk1dGnErnBW8"/"_id": $body.hits.events.0._id/] ## Check for malicious script loads [eql-ex-check-for-malicious-script-loads] @@ -167,6 +173,7 @@ GET /my-data-stream/_eql/search """ } ``` +% TEST[setup:atomic_red_regsvr32] The query matches an event, confirming `scrobj.dll` was loaded. @@ -203,6 +210,9 @@ The query matches an event, confirming `scrobj.dll` was loaded. } } ``` +% TESTRESPONSE[s/ ...\n/"is_partial": false, "is_running": false, "took": $body.took, "timed_out": false,/] +% TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.12.07-000001"/"_index": $body.hits.events.0._index/] +% TESTRESPONSE[s/"_id": "ol5MJXMBMk1dGnErnBW8"/"_id": $body.hits.events.0._id/] ## Determine the likelihood of success [eql-ex-detemine-likelihood-of-success] @@ -226,6 +236,7 @@ GET /my-data-stream/_eql/search """ } ``` +% TEST[setup:atomic_red_regsvr32] The query matches a sequence, indicating the attack likely succeeded. @@ -332,4 +343,9 @@ The query matches a sequence, indicating the attack likely succeeded. } } ``` +% TESTRESPONSE[s/ ...\n/"is_partial": false, "is_running": false, "took": $body.took, "timed_out": false,/] +% TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.12.07-000001"/"_index": $body.hits.sequences.0.events.0._index/] +% TESTRESPONSE[s/"_id": "gl5MJXMBMk1dGnErnBW8"/"_id": $body.hits.sequences.0.events.0._id/] +% TESTRESPONSE[s/"_id": "ol5MJXMBMk1dGnErnBW8"/"_id": $body.hits.sequences.0.events.1._id/] +% TESTRESPONSE[s/"_id": "EF5MJXMBMk1dGnErnBa9"/"_id": $body.hits.sequences.0.events.2._id/] diff --git a/explore-analyze/query-filter/languages/sql-async.md b/explore-analyze/query-filter/languages/sql-async.md index 1d7d5c46d7..a94b38946a 100644 --- a/explore-analyze/query-filter/languages/sql-async.md +++ b/explore-analyze/query-filter/languages/sql-async.md @@ -20,6 +20,9 @@ POST _sql?format=json "fetch_size": 5 } ``` +% TEST[skip:waiting on https://github.com/elastic/elasticsearch/issues/106158] +% TEST[setup:library] +% TEST[s/"wait_for_completion_timeout": "2s"/"wait_for_completion_timeout": "0"/] If the search doesn’t finish within this period, the search becomes async. The API returns: @@ -37,12 +40,17 @@ For CSV, TSV, and TXT responses, the API returns these values in the respective "rows": [ ] } ``` +% TESTRESPONSE[skip:waiting on https://github.com/elastic/elasticsearch/issues/106158] +% TESTRESPONSE[s/FnR0TDhyWUVmUmVtWXRWZER4MXZiNFEad2F5UDk2ZVdTVHV1S0xDUy00SklUdzozMTU=/$body.id/] +% TESTRESPONSE[s/"is_partial": true/"is_partial": $body.is_partial/] +% TESTRESPONSE[s/"is_running": true/"is_running": $body.is_running/] To check the progress of an async search, use the search ID with the [get async SQL search status API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-sql-get-async-status). ```console GET _sql/async/status/FnR0TDhyWUVmUmVtWXRWZER4MXZiNFEad2F5UDk2ZVdTVHV1S0xDUy00SklUdzozMTU= ``` +% TEST[skip: no access to search ID] If `is_running` and `is_partial` are `false`, the async search has finished with complete results. @@ -55,12 +63,16 @@ If `is_running` and `is_partial` are `false`, the async search has finished with "completion_status": 200 } ``` +% TESTRESPONSE[skip:waiting on https://github.com/elastic/elasticsearch/issues/106158] +% TESTRESPONSE[s/FnR0TDhyWUVmUmVtWXRWZER4MXZiNFEad2F5UDk2ZVdTVHV1S0xDUy00SklUdzozMTU=/$body.id/] +% TESTRESPONSE[s/"expiration_time_in_millis": 1611690295000/"expiration_time_in_millis": $body.expiration_time_in_millis/] To get the results, use the search ID with the [get async SQL search API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-sql-get-async). If the search is still running, specify how long you’d like to wait using `wait_for_completion_timeout`. You can also specify the response `format`. ```console GET _sql/async/FnR0TDhyWUVmUmVtWXRWZER4MXZiNFEad2F5UDk2ZVdTVHV1S0xDUy00SklUdzozMTU=?wait_for_completion_timeout=2s&format=json ``` +% TEST[skip: no access to search ID] ## Change the search retention period [sql-async-retention] @@ -76,18 +88,22 @@ POST _sql?format=json "fetch_size": 5 } ``` +% TEST[skip:waiting on https://github.com/elastic/elasticsearch/issues/106158] +% TEST[setup:library] You can use the get async SQL search API’s `keep_alive` parameter to later change the retention period. The new period starts after the request runs. ```console GET _sql/async/FmdMX2pIang3UWhLRU5QS0lqdlppYncaMUpYQ05oSkpTc3kwZ21EdC1tbFJXQToxOTI=?keep_alive=5d&wait_for_completion_timeout=2s&format=json ``` +% TEST[skip: no access to search ID] Use the [delete async SQL search API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-sql-delete-async) to delete an async search before the `keep_alive` period ends. If the search is still running, {{es}} cancels it. ```console DELETE _sql/async/delete/FmdMX2pIang3UWhLRU5QS0lqdlppYncaMUpYQ05oSkpTc3kwZ21EdC1tbFJXQToxOTI= ``` +% TEST[skip: no access to search ID] ## Store synchronous SQL searches [sql-store-searches] @@ -103,6 +119,8 @@ POST _sql?format=json "fetch_size": 5 } ``` +% TEST[skip:waiting on https://github.com/elastic/elasticsearch/issues/106158] +% TEST[setup:library] If `is_partial` and `is_running` are `false`, the search was synchronous and returned complete results. @@ -116,6 +134,11 @@ If `is_partial` and `is_running` are `false`, the search was synchronous and ret "cursor": ... } ``` +% TESTRESPONSE[skip:waiting on https://github.com/elastic/elasticsearch/issues/106158] +% TESTRESPONSE[s/Fnc5UllQdUVWU0NxRFNMbWxNYXplaFEaMUpYQ05oSkpTc3kwZ21EdC1tbFJXQTo0NzA=/$body.id/] +% TESTRESPONSE[s/"rows": .../"rows": $body.rows/] +% TESTRESPONSE[s/"columns": .../"columns": $body.columns/] +% TESTRESPONSE[s/"cursor": .../"cursor": $body.cursor/] You can get the same results later using the search ID with the [get async SQL search API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-sql-get-async). diff --git a/explore-analyze/query-filter/languages/sql-data-types.md b/explore-analyze/query-filter/languages/sql-data-types.md index 1f3d5deff1..316178b74a 100644 --- a/explore-analyze/query-filter/languages/sql-data-types.md +++ b/explore-analyze/query-filter/languages/sql-data-types.md @@ -92,6 +92,7 @@ Consider the following `string` mapping: } } ``` +% NOTCONSOLE The following SQL query: diff --git a/explore-analyze/query-filter/languages/sql-getting-started.md b/explore-analyze/query-filter/languages/sql-getting-started.md index de1ed4f093..cd4d1e946c 100644 --- a/explore-analyze/query-filter/languages/sql-getting-started.md +++ b/explore-analyze/query-filter/languages/sql-getting-started.md @@ -29,6 +29,7 @@ POST /_sql?format=txt "query": "SELECT * FROM library WHERE release_date < '2000-01-01'" } ``` +% TEST[continued] Which should return something along the lines of: @@ -38,6 +39,8 @@ Which should return something along the lines of: Dan Simmons |Hyperion |482 |1989-05-26T00:00:00.000Z Frank Herbert |Dune |604 |1965-06-01T00:00:00.000Z ``` +% TESTRESPONSE[s/|/\|/ s/+/\+/] +% TESTRESPONSE[non_json] You can also use the [*SQL CLI*](sql-cli.md). There is a script to start it shipped in the Elasticsearch `bin` directory: diff --git a/explore-analyze/query-filter/languages/sql-pagination.md b/explore-analyze/query-filter/languages/sql-pagination.md index fc7567c95f..3320506b77 100644 --- a/explore-analyze/query-filter/languages/sql-pagination.md +++ b/explore-analyze/query-filter/languages/sql-pagination.md @@ -16,6 +16,8 @@ POST /_sql?format=json "cursor": "sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWYUpOYklQMHhRUEtld3RsNnFtYU1hQQ==:BAFmBGRhdGUBZgVsaWtlcwFzB21lc3NhZ2UBZgR1c2Vy9f///w8=" } ``` +% TEST[continued] +% TEST[s/sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWYUpOYklQMHhRUEtld3RsNnFtYU1hQQ==:BAFmBGRhdGUBZgVsaWtlcwFzB21lc3NhZ2UBZgR1c2Vy9f///w8=/$body.cursor/] Which looks like: @@ -31,6 +33,7 @@ Which looks like: "cursor" : "sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWODRMaXBUaVlRN21iTlRyWHZWYUdrdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl9f///w8=" } ``` +% TESTRESPONSE[s/sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWODRMaXBUaVlRN21iTlRyWHZWYUdrdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl9f///w8=/$body.cursor/] Note that the `columns` object is only part of the first page. @@ -44,6 +47,8 @@ POST /_sql/close "cursor": "sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWYUpOYklQMHhRUEtld3RsNnFtYU1hQQ==:BAFmBGRhdGUBZgVsaWtlcwFzB21lc3NhZ2UBZgR1c2Vy9f///w8=" } ``` +% TEST[continued] +% TEST[s/sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWYUpOYklQMHhRUEtld3RsNnFtYU1hQQ==:BAFmBGRhdGUBZgVsaWtlcwFzB21lc3NhZ2UBZgR1c2Vy9f///w8=/$body.cursor/] Which will like return the diff --git a/explore-analyze/query-filter/languages/sql-rest-columnar.md b/explore-analyze/query-filter/languages/sql-rest-columnar.md index 2b7c8e8fe6..0fb3a3a96c 100644 --- a/explore-analyze/query-filter/languages/sql-rest-columnar.md +++ b/explore-analyze/query-filter/languages/sql-rest-columnar.md @@ -20,6 +20,7 @@ POST /_sql?format=json "columnar": true } ``` +% TEST[setup:library] Which returns: @@ -40,6 +41,7 @@ Which returns: "cursor": "sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl+v///w8=" } ``` +% TESTRESPONSE[s/sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl+v///w8=/$body.cursor/] Any subsequent calls using a `cursor` still have to contain the `columnar` parameter to preserve the orientation, meaning the initial query will not *remember* the columnar option. @@ -50,6 +52,8 @@ POST /_sql?format=json "columnar": true } ``` +% TEST[continued] +% TEST[s/sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl+v///w8=/$body.cursor/] Which looks like: @@ -64,4 +68,5 @@ Which looks like: "cursor": "46ToAwFzQERYRjFaWEo1UVc1a1JtVjBZMmdCQUFBQUFBQUFBQUVXWjBaNlFXbzNOV0pVY21Wa1NUZDJhV2t3V2xwblp3PT3/////DwQBZgZhdXRob3IBBHRleHQAAAFmBG5hbWUBBHRleHQAAAFmCnBhZ2VfY291bnQBBGxvbmcBAAFmDHJlbGVhc2VfZGF0ZQEIZGF0ZXRpbWUBAAEP" } ``` +% TESTRESPONSE[s/46ToAwFzQERYRjFaWEo1UVc1a1JtVjBZMmdCQUFBQUFBQUFBQUVXWjBaNlFXbzNOV0pVY21Wa1NUZDJhV2t3V2xwblp3PT3/////DwQBZgZhdXRob3IBBHRleHQAAAFmBG5hbWUBBHRleHQAAAFmCnBhZ2VfY291bnQBBGxvbmcBAAFmDHJlbGVhc2VfZGF0ZQEIZGF0ZXRpbWUBAAEP/$body.cursor/] diff --git a/explore-analyze/query-filter/languages/sql-rest-filtering.md b/explore-analyze/query-filter/languages/sql-rest-filtering.md index 3136e2587a..1cadf75bae 100644 --- a/explore-analyze/query-filter/languages/sql-rest-filtering.md +++ b/explore-analyze/query-filter/languages/sql-rest-filtering.md @@ -25,6 +25,7 @@ POST /_sql?format=txt "fetch_size": 5 } ``` +% TEST[setup:library] Which returns: @@ -33,6 +34,8 @@ Which returns: ---------------+------------------------------------+---------------+------------------------ Douglas Adams |The Hitchhiker's Guide to the Galaxy|180 |1979-10-12T00:00:00.000Z ``` +% TESTRESPONSE[s/|/\|/ s/+/\+/] +% TESTRESPONSE[non_json] ::::{tip} A useful and less obvious usage for standard Query DSL filtering is to search documents by a specific [routing key](elasticsearch://reference/elasticsearch/rest-apis/search-shard-routing.md#search-routing). Because Elasticsearch SQL does not support a `routing` parameter, one can specify a [`terms` filter for the `_routing` field](elasticsearch://reference/elasticsearch/mapping-reference/mapping-routing-field.md) instead: @@ -48,6 +51,7 @@ POST /_sql?format=txt } } ``` +% TEST[setup:library] :::: diff --git a/explore-analyze/query-filter/languages/sql-rest-format.md b/explore-analyze/query-filter/languages/sql-rest-format.md index 26202c7de5..7d37e549e7 100644 --- a/explore-analyze/query-filter/languages/sql-rest-format.md +++ b/explore-analyze/query-filter/languages/sql-rest-format.md @@ -43,6 +43,7 @@ POST /_sql?format=csv "fetch_size": 5 } ``` +% TEST[setup:library] which returns: @@ -54,6 +55,7 @@ Frank Herbert,Dune,604,1965-06-01T00:00:00.000Z Alastair Reynolds,Revelation Space,585,2000-03-15T00:00:00.000Z James S.A. Corey,Leviathan Wakes,561,2011-06-02T00:00:00.000Z ``` +% TESTRESPONSE[non_json] or: @@ -64,6 +66,7 @@ POST /_sql?format=csv&delimiter=%3b "fetch_size": 5 } ``` +% TEST[setup:library] which returns: @@ -75,6 +78,7 @@ Frank Herbert;Dune;604;1965-06-01T00:00:00.000Z Alastair Reynolds;Revelation Space;585;2000-03-15T00:00:00.000Z James S.A. Corey;Leviathan Wakes;561;2011-06-02T00:00:00.000Z ``` +% TESTRESPONSE[non_json] ## JSON [_json] @@ -86,6 +90,7 @@ POST /_sql?format=json "fetch_size": 5 } ``` +% TEST[setup:library] Which returns: @@ -107,6 +112,7 @@ Which returns: "cursor": "sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl+v///w8=" } ``` +% TESTRESPONSE[s/sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl+v///w8=/$body.cursor/] ## TSV [_tsv] @@ -118,6 +124,7 @@ POST /_sql?format=tsv "fetch_size": 5 } ``` +% TEST[setup:library] Which returns: @@ -129,6 +136,8 @@ Frank Herbert Dune 604 1965-06-01T00:00:00.000Z Alastair Reynolds Revelation Space 585 2000-03-15T00:00:00.000Z James S.A. Corey Leviathan Wakes 561 2011-06-02T00:00:00.000Z ``` +% TESTRESPONSE[s/\t/ /] +% TESTRESPONSE[non_json] ## TXT [_txt] @@ -140,6 +149,7 @@ POST /_sql?format=txt "fetch_size": 5 } ``` +% TEST[setup:library] Which returns: @@ -152,6 +162,8 @@ Frank Herbert |Dune |604 |1965-06-01T00:00:00.000Z Alastair Reynolds|Revelation Space |585 |2000-03-15T00:00:00.000Z James S.A. Corey |Leviathan Wakes |561 |2011-06-02T00:00:00.000Z ``` +% TESTRESPONSE[s/|/\|/ s/+/\+/] +% TESTRESPONSE[non_json] ## YAML [_yaml] @@ -163,6 +175,7 @@ POST /_sql?format=yaml "fetch_size": 5 } ``` +% TEST[setup:library] Which returns: @@ -199,5 +212,6 @@ rows: - "2011-06-02T00:00:00.000Z" cursor: "sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl+v///w8=" ``` +% TESTRESPONSE[s/sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl+v///w8=/$body.cursor/] diff --git a/explore-analyze/query-filter/languages/sql-rest-overview.md b/explore-analyze/query-filter/languages/sql-rest-overview.md index 41f500d15e..a949e8bc50 100644 --- a/explore-analyze/query-filter/languages/sql-rest-overview.md +++ b/explore-analyze/query-filter/languages/sql-rest-overview.md @@ -16,6 +16,7 @@ POST /_sql?format=txt "query": "SELECT * FROM library ORDER BY page_count DESC LIMIT 5" } ``` +% TEST[setup:library] Which returns: @@ -28,6 +29,8 @@ Frank Herbert |Dune |604 |1965-06-01T00:00:00.000Z Alastair Reynolds|Revelation Space |585 |2000-03-15T00:00:00.000Z James S.A. Corey |Leviathan Wakes |561 |2011-06-02T00:00:00.000Z ``` +% TESTRESPONSE[s/|/\|/ s/+/\+/] +% TESTRESPONSE[non_json] ::::{admonition} Using Kibana Console :class: tip diff --git a/explore-analyze/query-filter/languages/sql-rest-params.md b/explore-analyze/query-filter/languages/sql-rest-params.md index 7472dd9d18..06158d4da3 100644 --- a/explore-analyze/query-filter/languages/sql-rest-params.md +++ b/explore-analyze/query-filter/languages/sql-rest-params.md @@ -16,6 +16,7 @@ POST /_sql?format=txt "query": "SELECT YEAR(release_date) AS year FROM library WHERE page_count > 300 AND author = 'Frank Herbert' GROUP BY year HAVING COUNT(*) > 0" } ``` +% TEST[setup:library] or it can be done by extracting the values in a separate list of parameters and using question mark placeholders (`?`) in the query string: @@ -26,6 +27,7 @@ POST /_sql?format=txt "params": [300, "Frank Herbert", 0] } ``` +% TEST[setup:library] ::::{important} The recommended way of passing values to a query is with question mark placeholders, to avoid any attempts of hacking or SQL injection. diff --git a/explore-analyze/query-filter/languages/sql-runtime-fields.md b/explore-analyze/query-filter/languages/sql-runtime-fields.md index e9349e4f49..1f1871d8a9 100644 --- a/explore-analyze/query-filter/languages/sql-runtime-fields.md +++ b/explore-analyze/query-filter/languages/sql-runtime-fields.md @@ -28,6 +28,7 @@ POST _sql?format=txt """ } ``` +% TEST[setup:library] The API returns: @@ -36,4 +37,5 @@ The API returns: ---------------+---------------+---------------+------------------------+------------------- Frank Herbert |Dune |604 |1965-06-01T00:00:00.000Z|TUESDAY ``` +% TESTRESPONSE[non_json] diff --git a/explore-analyze/query-filter/languages/sql-translate.md b/explore-analyze/query-filter/languages/sql-translate.md index dbd6dac7f2..0ac5f43491 100644 --- a/explore-analyze/query-filter/languages/sql-translate.md +++ b/explore-analyze/query-filter/languages/sql-translate.md @@ -17,6 +17,7 @@ POST /_sql/translate "fetch_size": 10 } ``` +% TEST[setup:library] Which returns: diff --git a/explore-analyze/scripting/dissect.md b/explore-analyze/scripting/dissect.md index 4630a5fd4a..6e212d525a 100644 --- a/explore-analyze/scripting/dissect.md +++ b/explore-analyze/scripting/dissect.md @@ -21,6 +21,7 @@ For example, let’s say you have log data with a `message` field that looks lik ```js "message" : "247.37.0.0 - - [30/Apr/2020:14:31:22 -0500] \"GET /images/hm_nbg.jpg HTTP/1.0\" 304 0" ``` +% NOTCONSOLE You assign variables to each part of the data to construct a successful dissect pattern. Remember, tell dissect *exactly* what you want you want to match on. @@ -31,6 +32,7 @@ The first part of the data looks like an IP address, so you can assign a variabl %{clientip} %{ident} %{auth} [%{@timestamp}] <2> ``` +% NOTCONSOLE 1. The first chunks of data from the `message` field 2. Dissect pattern to match on the selected data chunks @@ -43,12 +45,14 @@ Using that same logic, you can create variables for the remaining chunks of data "%{verb} %{request} HTTP/%{httpversion}" %{response} %{size} ``` +% NOTCONSOLE Combining the two patterns results in a dissect pattern that looks like this: ```js %{clientip} %{ident} %{auth} [%{@timestamp}] \"%{verb} %{request} HTTP/%{httpversion}\" %{status} %{size} ``` +% NOTCONSOLE Now that you have a dissect pattern, how do you test and use it? @@ -98,6 +102,7 @@ POST /_scripts/painless/_execute } } ``` +% TEST[continued] 1. Runtime fields require the `emit` method to return values. 2. Because the response code is an integer, use the `long_field` context. @@ -154,6 +159,7 @@ PUT my-index/_mappings } } ``` +% TEST[continued] After mapping the fields you want to retrieve, index a few records from your log data into {{es}}. The following request uses the [bulk API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-bulk) to index raw log data into `my-index`: @@ -174,6 +180,7 @@ POST /my-index/_bulk?refresh=true {"index":{}} {"timestamp":"2020-04-30T14:31:28-05:00","message":"not a valid apache log"} ``` +% TEST[continued] You can define a simple query to run a search for a specific HTTP response and return all related fields. Use the `fields` parameter of the search API to retrieve the `http.response` runtime field. @@ -188,6 +195,7 @@ GET my-index/_search "fields" : ["http.response"] } ``` +% TEST[continued] Alternatively, you can define the same runtime field but in the context of a search request. The runtime definition and the script are exactly the same as the one defined previously in the index mapping. Just copy that definition into the search request under the `runtime_mappings` section and include a query that matches on the runtime field. This query returns the same results as the search query previously defined for the `http.response` runtime field in your index mappings, but only in the context of this specific search: @@ -211,6 +219,8 @@ GET my-index/_search "fields" : ["http.response"] } ``` +% TEST[continued] +% TEST[s/_search/_search?filter_path=hits/] ```console-result { @@ -239,5 +249,6 @@ GET my-index/_search } } ``` +% TESTRESPONSE[s/"_id" : "D47UqXkBByC8cgZrkbOm"/"_id": $body.hits.hits.0._id/] diff --git a/explore-analyze/scripting/grok.md b/explore-analyze/scripting/grok.md index 2b5e389f89..aad88407c4 100644 --- a/explore-analyze/scripting/grok.md +++ b/explore-analyze/scripting/grok.md @@ -66,6 +66,7 @@ For example, if you’re working with Apache log data, you can use the `%{{COMMO "timestamp":"2020-04-30T14:30:17-05:00","message":"40.135.0.0 - - [30/Apr/2020:14:30:17 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736" ``` +% NOTCONSOLE To extract the IP address from the `message` field, you can write a Painless script that incorporates the `%{{COMMONAPACHELOG}}` syntax. You can test this script using the [`ip` field context](elasticsearch://reference/scripting-languages/painless/painless-api-examples.md#painless-runtime-ip) of the Painless execute API, but let’s use a runtime field instead. @@ -107,6 +108,7 @@ POST /my-index/_bulk?refresh {"index":{}} {"timestamp":"2020-04-30T14:31:28-05:00","message":"not a valid apache log"} ``` +% TEST[continued] ## Incorporate grok patterns and scripts in runtime fields [grok-patterns-runtime] @@ -127,6 +129,7 @@ PUT my-index/_mappings } } ``` +% TEST[continued] Alternatively, you can define the same runtime field but in the context of a search request. The runtime definition and the script are exactly the same as the one defined previously in the index mapping. Just copy that definition into the search request under the `runtime_mappings` section and include a query that matches on the runtime field. This query returns the same results as if you [defined a search query](#grok-pattern-results) for the `http.clientip` runtime field in your index mappings, but only in the context of this specific search: @@ -150,6 +153,7 @@ GET my-index/_search "fields" : ["http.clientip"] } ``` +% TEST[continued] ## Return calculated results [grok-pattern-results] @@ -167,6 +171,8 @@ GET my-index/_search "fields" : ["http.clientip"] } ``` +% TEST[continued] +% TEST[s/_search/_search?filter_path=hits/] The response includes the specific IP address indicated in your search query. The grok pattern within the Painless script extracted this value from the `message` field at runtime. @@ -197,5 +203,6 @@ The response includes the specific IP address indicated in your search query. Th } } ``` +% TESTRESPONSE[s/"_id" : "1iN2a3kBw4xTzEDqyYE0"/"_id": $body.hits.hits.0._id/] diff --git a/explore-analyze/scripting/modules-scripting-engine.md b/explore-analyze/scripting/modules-scripting-engine.md index 52093e8bd4..faa0ab9716 100644 --- a/explore-analyze/scripting/modules-scripting-engine.md +++ b/explore-analyze/scripting/modules-scripting-engine.md @@ -194,4 +194,5 @@ POST /_search } } ``` +% TEST[skip:we don’t have an expert script plugin installed to test this] diff --git a/explore-analyze/scripting/modules-scripting-using.md b/explore-analyze/scripting/modules-scripting-using.md index 085f7e0366..cbbddee24c 100644 --- a/explore-analyze/scripting/modules-scripting-using.md +++ b/explore-analyze/scripting/modules-scripting-using.md @@ -17,6 +17,7 @@ Wherever scripting is supported in the {{es}} APIs, the syntax follows the same "params": { ... } } ``` +% NOTCONSOLE `lang` : Specifies the language the script is written in. Defaults to `painless`. @@ -62,6 +63,7 @@ GET my-index-000001/_search } } ``` +% TEST[continued] 1. `script` object 2. `script` source @@ -96,6 +98,7 @@ You can compile up to 150 scripts per 5 minutes by default. For ingest contexts, ```js script.context.field.max_compilations_rate=100/10m ``` +% NOTCONSOLE ::::{important} If you compile too many unique scripts within a short time, {{es}} rejects the new dynamic scripts with a `circuit_breaking_exception` error. @@ -123,6 +126,7 @@ GET my-index-000001/_search } } ``` +% TEST[s/^/PUT my-index-000001\n/] Let’s look at a shortened version of the script to see what improvements it includes over the previous iteration: @@ -141,6 +145,7 @@ GET my-index-000001/_search } } ``` +% TEST[s/^/PUT my-index-000001\n/] This version of the script removes several components and simplifies the syntax significantly: @@ -180,6 +185,7 @@ You can retrieve that script by using the [get stored script API](https://www.el ```console GET _scripts/calculate-score ``` +% TEST[continued] To use the stored script in a query, include the script `id` in the `script` declaration: @@ -203,6 +209,8 @@ GET my-index-000001/_search } } ``` +% TEST[setup:my_index] +% TEST[continued] 1. `id` of the stored script @@ -212,6 +220,7 @@ To delete a stored script, submit a [delete stored script API](https://www.elast ```console DELETE _scripts/calculate-score ``` +% TEST[continued] ## Update documents with scripts [scripts-update-scripts] @@ -242,6 +251,7 @@ POST my-index-000001/_update/1 } } ``` +% TEST[continued] Similarly, you can use an update script to add a tag to the list of tags. Because this is just a list, the tag is added even it exists: @@ -257,6 +267,7 @@ POST my-index-000001/_update/1 } } ``` +% TEST[continued] You can also remove a tag from the list of tags. The `remove` method of a Java `List` is available in Painless. It takes the index of the element you want to remove. To avoid a possible runtime error, you first need to make sure the tag exists. If the list contains duplicates of the tag, this script just removes one occurrence. @@ -272,6 +283,7 @@ POST my-index-000001/_update/1 } } ``` +% TEST[continued] You can also add and remove fields from a document. For example, this script adds the field `new_field`: @@ -281,6 +293,7 @@ POST my-index-000001/_update/1 "script" : "ctx._source.new_field = 'value_of_new_field'" } ``` +% TEST[continued] Conversely, this script removes the field `new_field`: @@ -290,6 +303,7 @@ POST my-index-000001/_update/1 "script" : "ctx._source.remove('new_field')" } ``` +% TEST[continued] Instead of updating the document, you can also change the operation that is executed from within the script. For example, this request deletes the document if the `tags` field contains `green`. Otherwise it does nothing (`noop`): @@ -305,6 +319,7 @@ POST my-index-000001/_update/1 } } ``` +% TEST[continued] diff --git a/explore-analyze/scripting/scripting-field-extraction.md b/explore-analyze/scripting/scripting-field-extraction.md index ad3490e902..6e329e3cfe 100644 --- a/explore-analyze/scripting/scripting-field-extraction.md +++ b/explore-analyze/scripting/scripting-field-extraction.md @@ -53,6 +53,7 @@ POST /my-index/_bulk?refresh {"index":{}} {"timestamp":"2020-04-30T14:31:28-05:00","message":"not a valid apache log"} ``` +% TEST[continued] ## Extract an IP address from a log message (Grok) [field-extraction-ip] @@ -75,6 +76,7 @@ PUT my-index/_mappings } } ``` +% TEST[continued] 1. This condition ensures that the script doesn’t emit anything even if the pattern of the message doesn’t match. @@ -92,6 +94,8 @@ GET my-index/_search "fields" : ["http.clientip"] } ``` +% TEST[continued] +% TEST[s/_search/_search?filter_path=hits/] The response includes documents where the value for `http.clientip` matches `40.135.0.0`. @@ -122,6 +126,7 @@ The response includes documents where the value for `http.clientip` matches `40. } } ``` +% TESTRESPONSE[s/"_id" : "Rq-ex3gBA_A0V6dYGLQ7"/"_id": $body.hits.hits.0._id/] ## Parse a string to extract part of a field (Dissect) [field-extraction-parse] @@ -133,6 +138,7 @@ For example, the log data at the start of this section includes a `message` fiel ```js "message" : "247.37.0.0 - - [30/Apr/2020:14:31:22 -0500] \"GET /images/hm_nbg.jpg HTTP/1.0\" 304 0" ``` +% NOTCONSOLE You can define a dissect pattern in a runtime field to extract the [HTTP response code](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status), which is `304` in the previous example. @@ -150,6 +156,7 @@ PUT my-index/_mappings } } ``` +% TEST[continued] You can then run a query to retrieve a specific HTTP response using the `http.response` runtime field: @@ -164,6 +171,8 @@ GET my-index/_search "fields" : ["http.response"] } ``` +% TEST[continued] +% TEST[s/_search/_search?filter_path=hits/] The response includes a single document where the HTTP response is `304`: @@ -194,6 +203,7 @@ The response includes a single document where the HTTP response is `304`: } } ``` +% TESTRESPONSE[s/"_id" : "Sq-ex3gBA_A0V6dYGLQ7"/"_id": $body.hits.hits.0._id/] ## Split values in a field by a separator (Dissect) [field-extraction-split] @@ -205,6 +215,7 @@ For example, let’s say you have a bunch of garbage collection (gc) log data fr ```txt [2021-04-27T16:16:34.699+0000][82460][gc,heap,exit] class space used 266K, capacity 384K, committed 384K, reserved 1048576K ``` +% NOTCONSOLE You only want to extract the `used`, `capacity`, and `committed` data, along with the associated values. Let’s index some a few documents containing log data to use as an example: @@ -272,6 +283,7 @@ GET my-index/_search "fields" : ["gc_size"] } ``` +% TEST[continued] The response includes the data from the `gc_size` field, formatted exactly as you defined it in the dissect pattern! @@ -341,4 +353,6 @@ The response includes the data from the `gc_size` field, formatted exactly as yo } } ``` +% TESTRESPONSE[s/"took" : 2/"took": "$body.took"/] +% TESTRESPONSE[s/"_id" : "GXx3H3kBKGE42WRNlddJ"/"_id": $body.hits.hits.0._id/] diff --git a/explore-analyze/scripting/scripts-search-speed.md b/explore-analyze/scripting/scripts-search-speed.md index ba2bbd18f2..1bc994b4f6 100644 --- a/explore-analyze/scripting/scripts-search-speed.md +++ b/explore-analyze/scripting/scripts-search-speed.md @@ -56,6 +56,7 @@ GET /my_test_scores/_search ] } ``` +% TEST[s/^/PUT my_test_scores\n/] If you’re searching a small index, then including the script as part of your search query can be a good solution. If you want to make search faster, you can perform this calculation during ingest and index the sum to a field instead. @@ -71,6 +72,7 @@ PUT /my_test_scores/_mapping } } ``` +% TEST[continued] Next, use an [ingest pipeline](../../manage-data/ingest/transform-enrich/ingest-pipelines.md) containing the [script processor](elasticsearch://reference/ingestion-tools/enrich-processor/script-processor.md) to calculate the sum of `math_score` and `verbal_score` and index it in the `total_score` field. @@ -87,6 +89,7 @@ PUT _ingest/pipeline/my_test_scores_pipeline ] } ``` +% TEST[continued] To update existing data, use this pipeline to [reindex](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-reindex) any documents from `my_test_scores` to a new index named `my_test_scores_2`. @@ -102,6 +105,7 @@ POST /_reindex } } ``` +% TEST[continued] Continue using the pipeline to index any new documents to `my_test_scores_2`. @@ -114,6 +118,7 @@ POST /my_test_scores_2/_doc/?pipeline=my_test_scores_pipeline "verbal_score": 800 } ``` +% TEST[continued] These changes slow the index process, but allow for faster searches. Instead of using a script, you can sort searches made on `my_test_scores_2` using the `total_score` field. The response is near real-time! Though this process slows ingest time, it greatly increases queries at search time. @@ -134,4 +139,5 @@ GET /my_test_scores_2/_search ] } ``` +% TEST[continued] diff --git a/explore-analyze/transforms/transform-api-quickref.md b/explore-analyze/transforms/transform-api-quickref.md index 49c94c5f00..162d235bc9 100644 --- a/explore-analyze/transforms/transform-api-quickref.md +++ b/explore-analyze/transforms/transform-api-quickref.md @@ -13,6 +13,7 @@ All {{transform}} endpoints have the following base: ```js _transform/ ``` +% NOTCONSOLE * [Create {{transforms}}](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-transform-put-transform) * [Delete {{transforms}}](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-transform-delete-transform) diff --git a/explore-analyze/transforms/transform-examples.md b/explore-analyze/transforms/transform-examples.md index 5feffc7ba3..7b68b2aa07 100644 --- a/explore-analyze/transforms/transform-examples.md +++ b/explore-analyze/transforms/transform-examples.md @@ -54,6 +54,7 @@ POST _transform/_preview } } ``` +% TEST[skip:setup kibana sample data] 1. The destination index for the {{transform}}. It is ignored by `_preview`. 2. Two `group_by` fields is selected. This means the {{transform}} contains a unique row per `user` and `customer_id` combination. Within this data set, both these fields are unique. By including both in the {{transform}}, it gives more context to the final results. @@ -80,6 +81,7 @@ The preview {{transforms}} API enables you to see the layout of the {{transform} ] } ``` +% NOTCONSOLE ::::: @@ -133,6 +135,7 @@ POST _transform/_preview } } ``` +% TEST[skip:setup kibana sample data] 1. Filter the source data to select only flights that are not cancelled. 2. The destination index for the {{transform}}. It is ignored by `_preview`. @@ -155,6 +158,7 @@ The preview shows you that the new index would contain data like this for each c ] } ``` +% NOTCONSOLE This {{transform}} makes it easier to answer questions such as: @@ -221,6 +225,7 @@ PUT _transform/suspicious_client_ips } } ``` +% TEST[skip:setup kibana sample data] 1. The destination index for the {{transform}}. 2. Configures the {{transform}} to run continuously. It uses the `timestamp` field to synchronize the source and destination indices. The worst case ingestion delay is 60 seconds. @@ -233,12 +238,14 @@ After you create the {{transform}}, you must start it: ```console POST _transform/suspicious_client_ips/_start ``` +% TEST[skip:setup kibana sample data] Shortly thereafter, the first results should be available in the destination index: ```console GET sample_weblogs_by_clientip/_search ``` +% TEST[skip:setup kibana sample data] The search result shows you data like this for each client IP: @@ -272,6 +279,7 @@ The search result shows you data like this for each client IP: } ] ``` +% NOTCONSOLE ::::{note} Like other Kibana sample data sets, the web log sample dataset contains timestamps relative to when you installed it, including timestamps in the future. The {{ctransform}} will pick up the data points once they are in the past. If you installed the web log sample dataset some time ago, you can uninstall and reinstall it and the timestamps will change. @@ -340,6 +348,7 @@ PUT _transform/last-log-from-clientip } } ``` +% TEST[skip:setup kibana sample data] 1. Specifies the field for grouping the data. 2. Specifies the date field that is used for sorting the data. @@ -352,6 +361,7 @@ After you create the {{transform}}, start it: ```console POST _transform/last-log-from-clientip/_start ``` +% TEST[skip:setup kibana sample data] :::: @@ -360,6 +370,7 @@ After the {{transform}} processes the data, search the destination index: ```console GET last-log-from-clientip/_search ``` +% TEST[skip:setup kibana sample data] The search result shows you data like this for each client IP: @@ -408,6 +419,7 @@ The search result shows you data like this for each client IP: } } ``` +% NOTCONSOLE This {{transform}} makes it easier to answer questions such as: @@ -459,6 +471,7 @@ POST _transform/_preview } } ``` +% TEST[skip:setup kibana sample data] 1. The data is grouped by a date histogram of the time field with a one hour interval. 2. Calculates the maximum value of the `bytes` field. @@ -503,6 +516,7 @@ The API call above returns a response similar to this: ] } ``` +% NOTCONSOLE ## Getting customer name and email address by customer ID [example-customer-names] @@ -547,6 +561,7 @@ POST _transform/_preview } } ``` +% TEST[skip:setup kibana sample data] 1. The data is grouped by a `terms` aggregation on the `customer_id` field. 2. Specifies the fields to return (email and name fields) in a descending order by the order date. @@ -584,3 +599,4 @@ The API returns a response that is similar to this: ] } ``` +% NOTCONSOLE diff --git a/explore-analyze/transforms/transform-painless-examples.md b/explore-analyze/transforms/transform-painless-examples.md index 962bff1a59..5cb458981e 100644 --- a/explore-analyze/transforms/transform-painless-examples.md +++ b/explore-analyze/transforms/transform-painless-examples.md @@ -60,6 +60,7 @@ This example uses a `scripted_metric` aggregation which is not supported on {{es } } ``` +% NOTCONSOLE 1. The `init_script` creates a long type `timestamp_latest` and a string type `last_doc` in the `state` object. 2. The `map_script` defines `current_date` based on the timestamp of the document, then compares `current_date` with `state.timestamp_latest`, finally returns `state.last_doc` from the shard. By using `new HashMap(...)` you copy the source document, this is important whenever you want to pass the full source object from one phase to the next. @@ -93,6 +94,7 @@ You can retrieve the last value in a similar way: } } ``` +% NOTCONSOLE ### Getting top hits by using stored scripts [top-hits-stored-scripts] @@ -203,6 +205,7 @@ This snippet shows how to extract time based features by using Painless in a {{t ... } ``` +% NOTCONSOLE 1. Name of the aggregation. 2. Contains the Painless script that returns the hour of the day. @@ -255,6 +258,7 @@ PUT _transform/data_log } } ``` +% TEST[skip:setup kibana sample data] 1. To define the length of the sessions, we use a bucket script. 2. The bucket path is a map of script variables and their associated path to the buckets you want to use for the variable. In this particular case, `min` and `max` are variables mapped to `time_frame.gte.value` and `time_frame.lte.value`. @@ -300,6 +304,7 @@ This example uses a `scripted_metric` aggregation which is not supported on {{es ... } ``` +% NOTCONSOLE 1. The `aggregations` object of the {{transform}} that contains all aggregations. 2. Object of the `scripted_metric` aggregation. @@ -362,6 +367,7 @@ POST _transform/_preview } } ``` +% TEST[skip:setup kibana sample data] 1. The indices referenced in the `source` object are compared to each other. 2. The `dest` index contains the results of the comparison. @@ -411,6 +417,7 @@ This example shows how to derive multiple features from a single transaction. Le } ... ``` +% NOTCONSOLE :::: @@ -487,6 +494,7 @@ POST _transform/_preview } } ``` +% NOTCONSOLE 1. The data is grouped by `sessionid`. 2. The aggregations counts the number of paths and enumerate the viewed pages during the session. @@ -526,3 +534,4 @@ The API call results in a similar response: } ... ``` +% NOTCONSOLE diff --git a/explore-analyze/transforms/transform-scale.md b/explore-analyze/transforms/transform-scale.md index 7b829b5570..ab351ccb38 100644 --- a/explore-analyze/transforms/transform-scale.md +++ b/explore-analyze/transforms/transform-scale.md @@ -65,6 +65,7 @@ Consider using [date math](elasticsearch://reference/elasticsearch/rest-apis/api ] }, ``` +% NOTCONSOLE ## 5. Optimize the sharding strategy for the source index (search) [optimize-shading-strategy] diff --git a/manage-data/data-store/aliases.md b/manage-data/data-store/aliases.md index 043961a36f..c5bbaad6fc 100644 --- a/manage-data/data-store/aliases.md +++ b/manage-data/data-store/aliases.md @@ -44,6 +44,7 @@ POST _aliases ] } ``` +% TEST[s/^/PUT _data_stream/logs-nginx.access-prod\n/] The API’s `index` and `indices` parameters support wildcards (`*`). Wildcard patterns that match both data streams and indices return an error. @@ -60,6 +61,7 @@ POST _aliases ] } ``` +% TEST[s/^/PUT _data_stream/logs-nginx.access-prod\n/] ## Remove an alias [remove-alias] @@ -79,6 +81,7 @@ POST _aliases ] } ``` +% TEST[continued] ## Multiple actions [multiple-actions] @@ -106,6 +109,7 @@ POST _aliases ] } ``` +% TEST[s/^/PUT _data_stream/logs-nginx.access-prod\nPUT _data_stream/logs-my_app-default\n/] ## Multiple action results [multiple-action-results] @@ -133,6 +137,7 @@ POST _aliases ] } ``` +% TEST[s/^/PUT /index1\nPUT /index2\n/] The API returns the following result: @@ -203,6 +208,8 @@ PUT _index_template/my-index-template } } ``` +% TEST[s/,\n "my-mappings",\n "my-settings"//] +% TEST[teardown:data_stream_cleanup] You can also specify index aliases in [create index API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-create) requests. @@ -224,18 +231,22 @@ To get a list of your cluster’s aliases, use the [get alias API](https://www.e ```console GET _alias ``` +% TEST[s/^/PUT _data_stream/logs-nginx.access-prod\nPUT logs-nginx.access-prod/_alias/logs\n/] Specify a data stream or index before `_alias` to view its aliases. ```console GET my-data-stream/_alias ``` +% TEST[s/^/PUT _data_stream/logs-nginx.access-prod\nPUT logs-nginx.access-prod/_alias/logs\n/] +% TEST[s/my-data-stream/logs-nginx.access-prod/] Specify an alias after `_alias` to view its data streams or indices. ```console GET _alias/logs ``` +% TEST[s/^/PUT _data_stream/logs-nginx.access-prod\nPUT logs-nginx.access-prod/_alias/logs\n/] ## Write index [write-index] @@ -262,6 +273,7 @@ POST _aliases ] } ``` +% TEST[s/^/PUT _data_stream/logs-nginx.access-prod\nPUT _data_stream/logs-my_app-default\n/] If an alias points to multiple indices or data streams and `is_write_index` isn’t set, the alias rejects write requests. If an index alias points to one index and `is_write_index` isn’t set, the index automatically acts as the write index. Data stream aliases don’t automatically set a write data stream, even if the alias points to one data stream. @@ -307,6 +319,7 @@ POST _aliases ] } ``` +% TEST[s/^/PUT my-index-2099.05.06-000001\n/] ::::{note} Filters are only applied when using the [Query DSL](../../explore-analyze/query-filter/languages/querydsl.md), and are not applied when [retrieving a document by ID](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-get). @@ -332,6 +345,7 @@ POST _aliases ] } ``` +% TEST[s/^/PUT my-index-2099.05.06-000001\n/] Use `index_routing` and `search_routing` to specify different routing values for indexing and search. If specified, these options overwrite the `routing` value for their respective operations. @@ -350,6 +364,7 @@ POST _aliases ] } ``` +% TEST[s/^/PUT my-index-2099.05.06-000001\n/] ## Remove an index [remove-index] @@ -368,3 +383,4 @@ POST _aliases ] } ``` +% TEST[s/^/PUT my-index-2099.05.06-000001\n/] diff --git a/manage-data/data-store/data-streams/logs-data-stream.md b/manage-data/data-store/data-streams/logs-data-stream.md index cccf0513f5..44b5dc6640 100644 --- a/manage-data/data-store/data-streams/logs-data-stream.md +++ b/manage-data/data-store/data-streams/logs-data-stream.md @@ -35,6 +35,7 @@ PUT _index_template/my-index-template "priority": 101 <2> } ``` +% TEST 1. The index mode setting. 2. The index template priority. By default, Elasticsearch ships with a `logs-*-*` index template with a priority of 100. To make sure your index template takes priority over the default `logs-*-*` template, set its `priority` to a number higher than 100. For more information, see [Avoid index pattern collisions](../templates.md#avoid-index-pattern-collisions). diff --git a/manage-data/data-store/data-streams/modify-data-stream.md b/manage-data/data-store/data-streams/modify-data-stream.md index 82d70cac53..7adc165b71 100644 --- a/manage-data/data-store/data-streams/modify-data-stream.md +++ b/manage-data/data-store/data-streams/modify-data-stream.md @@ -498,4 +498,5 @@ POST _aliases ] } ``` +% TEST[s/^/PUT _data_stream/logs-nginx.access-prod\nPUT _data_stream/logs-my_app-default\n/] diff --git a/manage-data/data-store/data-streams/reindex-tsds.md b/manage-data/data-store/data-streams/reindex-tsds.md index 69384cee58..d535b4c8e9 100644 --- a/manage-data/data-store/data-streams/reindex-tsds.md +++ b/manage-data/data-store/data-streams/reindex-tsds.md @@ -83,6 +83,7 @@ POST /_index_template/1 "data_stream": {} } ``` +% TEST[skip: not expected to match the sample below] A possible output of `/k8s/_settings` looks like: @@ -144,6 +145,7 @@ A possible output of `/k8s/_settings` looks like: } } ``` +% NOTCONSOLE To reindex this TSDS, do not to re-use its index template in the destination data stream, to avoid impacting its functionality. Instead, clone the template of the source TSDS and apply the following modifications: @@ -199,6 +201,7 @@ POST /_index_template/2 "data_stream": {} } ``` +% TEST[continued] ## Reindex [tsds-reindex-op] @@ -217,6 +220,7 @@ POST /_reindex } } ``` +% TEST[continued] ## Restore the destination index template [tsds-reindex-restore] @@ -258,12 +262,14 @@ POST /_component_template/destination_template } } ``` +% TEST[continued] Next, Invoke the `rollover` api on the destination data stream without any conditions set. ```console POST /k9s/_rollover/ ``` +% TEST[continued] This creates a new backing index with the updated index settings. The destination data stream is now ready to accept new documents. diff --git a/manage-data/data-store/data-streams/run-downsampling-manually.md b/manage-data/data-store/data-streams/run-downsampling-manually.md index 4ac5f2b1e8..43df992f50 100644 --- a/manage-data/data-store/data-streams/run-downsampling-manually.md +++ b/manage-data/data-store/data-streams/run-downsampling-manually.md @@ -223,6 +223,7 @@ PUT _ingest/pipeline/my-timestamp-pipeline ] } ``` +% TEST[continued] Next, use a bulk API request to automatically create your TSDS and index a set of ten documents: @@ -249,12 +250,15 @@ PUT /my-data-stream/_bulk?refresh&pipeline=my-timestamp-pipeline {"create": {}} {"@timestamp":"2022-06-21T15:38:30Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":40018,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":1062428344},"usage":{"bytes":265142477,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":2294743},"rss":{"bytes":340623},"pagefaults":224530,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}} ``` +% TEST[continued] You can use the search API to check if the documents have been indexed correctly: ```console GET /my-data-stream/_search ``` +% TEST[skip:Because we’ve skipped the previous steps] +% TEST[continued] Run the following aggregation on the data to calculate some interesting statistics: @@ -296,6 +300,8 @@ GET /my-data-stream/_search } } ``` +% TEST[skip:Because we’ve skipped the previous steps] +% TEST[continued] ## Downsample the TSDS [downsampling-manual-run] @@ -305,6 +311,7 @@ A TSDS can’t be downsampled directly. You need to downsample its backing indic ```console GET /_data_stream/my-data-stream ``` +% TEST[continued] This returns: @@ -346,6 +353,11 @@ This returns: ] } ``` +% TESTRESPONSE[s/".ds-my-data-stream-2023.07.26-000001"/$body.data_streams.0.indices.0.index_name/] +% TESTRESPONSE[s/"ltOJGmqgTVm4T-Buoe7Acg"/$body.data_streams.0.indices.0.index_uuid/] +% TESTRESPONSE[s/"2023-07-26T09:26:42.000Z"/$body.data_streams.0.time_series.temporal_ranges.0.start/] +% TESTRESPONSE[s/"2023-07-26T13:26:42.000Z"/$body.data_streams.0.time_series.temporal_ranges.0.end/] +% TESTRESPONSE[s/"replicated": false/"replicated": false,"failure_store":{"enabled": false, "indices": [], "rollover_on_write": true}/] 1. The backing index for this data stream. @@ -357,6 +369,7 @@ Roll over the TSDS using the [rollover API](https://www.elastic.co/docs/api/doc/ ```console POST /my-data-stream/_rollover/ ``` +% TEST[continued] Copy the name of the `old_index` from the response. In the following steps, replace the index name with that of your `old_index`. @@ -365,6 +378,7 @@ The old index needs to be set to read-only mode. Run the following request: ```console PUT /.ds-my-data-stream-2023.07.26-000001/_block/write ``` +% TEST[skip:We don’t know the index name at test time] Next, use the [downsample API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-downsample) to downsample the index, setting the time series interval to one hour: @@ -374,6 +388,7 @@ POST /.ds-my-data-stream-2023.07.26-000001/_downsample/.ds-my-data-stream-2023.0 "fixed_interval": "1h" } ``` +% TEST[skip:We don’t know the index name at test time] Now you can [modify the data stream](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-modify-data-stream), and replace the original index with the downsampled one: @@ -396,6 +411,7 @@ POST _data_stream/_modify ] } ``` +% TEST[skip:We don’t know the index name at test time] You can now delete the old backing index. But be aware this will delete the original data. Don’t delete the index if you may need the original data in the future. @@ -517,6 +533,7 @@ The TSDS with the new downsampled backing index contains just one document. For } } ``` +% TEST[skip:Because we’ve skipped the previous step] Re-run the earlier aggregation. Even though the aggregation runs on the downsampled TSDS that only contains 1 document, it returns the same results as the earlier aggregation on the original TSDS. diff --git a/manage-data/data-store/data-streams/run-downsampling-using-data-stream-lifecycle.md b/manage-data/data-store/data-streams/run-downsampling-using-data-stream-lifecycle.md index a219294294..7f285394cb 100644 --- a/manage-data/data-store/data-streams/run-downsampling-using-data-stream-lifecycle.md +++ b/manage-data/data-store/data-streams/run-downsampling-using-data-stream-lifecycle.md @@ -219,6 +219,7 @@ PUT /datastream/_bulk?refresh {"create": {}} {"@timestamp":"2022-06-21T15:38:30Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":40018,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":1062428344},"usage":{"bytes":265142477,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":2294743},"rss":{"bytes":340623},"pagefaults":224530,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}} ``` +% TEST[skip: timestamp values won’t match an accepted range in the TSDS] ## View current state of data stream [downsampling-dsl-view-data-stream-state] @@ -228,6 +229,8 @@ Now that you’ve created and added documents to the data stream, check to confi ```console GET _data_stream ``` +% TEST[skip: temporal_ranges and index names won’t match] +% TEST[skip: temporal_ranges and index names won’t match] If the data stream lifecycle policy has not yet been applied, your results will be like the following. Note the original `index_name`: `.ds-datastream-2024.04.29-000001`. @@ -276,12 +279,16 @@ If the data stream lifecycle policy has not yet been applied, your results will ] } ``` +% TEST[skip: some fields are removed for brevity] +% TEST[continued] Next, run a search query: ```console GET datastream/_search ``` +% TEST[continued] +% TEST[skip: timestamp values won’t match] The query returns your ten newly added documents. @@ -302,6 +309,8 @@ The query returns your ten newly added documents. }, ... ``` +% TEST[skip: some fields are removed for brevity] +% TEST[continued] ## Roll over the data stream [downsampling-dsl-rollover] @@ -313,6 +322,7 @@ Roll over the data stream using the [rollover API](https://www.elastic.co/docs/a ```console POST /datastream/_rollover/ ``` +% TEST[continued] ## View downsampling results [downsampling-dsl-view-results] @@ -347,6 +357,8 @@ After the data stream lifecycle action was executed, original `.ds-datastream-20 ], ... ``` +% TEST[skip: some fields are removed for brevity] +% TEST[continued] Run a search query on the datastream (note that when querying downsampled indices there are [a few nuances to be aware of](./downsampling-time-series-data-stream.md#querying-downsampled-indices-notes)). @@ -462,12 +474,15 @@ The new downsampled index contains just one document that includes the `min`, `m } } ``` +% TEST[skip: timestamp values won’t match] +% TEST[continued] Use the [data stream stats API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-data-streams-stats-1) to get statistics for the data stream, including the storage size. ```console GET /_data_stream/datastream/_stats?human=true ``` +% TEST[continued] ```console-result { @@ -491,5 +506,7 @@ GET /_data_stream/datastream/_stats?human=true ] } ``` +% TEST[skip: exact size may be different] +% TEST[continued] This example demonstrates how downsampling works as part of a data stream lifecycle to reduce the storage size of metrics data as it becomes less current and less frequently queried. diff --git a/manage-data/data-store/data-streams/run-downsampling-with-ilm.md b/manage-data/data-store/data-streams/run-downsampling-with-ilm.md index 5051279516..4b49144246 100644 --- a/manage-data/data-store/data-streams/run-downsampling-with-ilm.md +++ b/manage-data/data-store/data-streams/run-downsampling-with-ilm.md @@ -211,6 +211,7 @@ PUT _index_template/datastream_template } } ``` +% TEST[continued] ## Ingest time series data [downsampling-ilm-ingest-data] @@ -242,6 +243,7 @@ PUT /datastream/_bulk?refresh {"create": {}} {"@timestamp":"2022-06-21T15:38:30Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":40018,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":1062428344},"usage":{"bytes":265142477,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":2294743},"rss":{"bytes":340623},"pagefaults":224530,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}} ``` +% TEST[skip: The @timestamp value won’t match an accepted range in the TSDS] ## View the results [downsampling-ilm-view-results] @@ -251,6 +253,8 @@ Now that you’ve created and added documents to the data stream, check to confi ```console GET _data_stream ``` +% TEST[skip: The @timestamp value won’t match an accepted range in the TSDS] +% TEST[skip: The @timestamp value won’t match an accepted range in the TSDS] If the ILM policy has not yet been applied, your results will be like the following. Note the original `index_name`: `.ds-datastream--000001`. @@ -293,12 +297,16 @@ If the ILM policy has not yet been applied, your results will be like the follow ] } ``` +% TEST[skip:todo] +% TEST[continued] Next, run a search query: ```console GET datastream/_search ``` +% TEST[continued] +% TEST[skip: The @timestamp value won’t match an accepted range in the TSDS] The query returns your ten newly added documents. @@ -319,6 +327,8 @@ The query returns your ten newly added documents. }, ... ``` +% TEST[skip:todo] +% TEST[continued] By default, index lifecycle management checks every ten minutes for indices that meet policy criteria. Wait for about ten minutes (maybe brew up a quick coffee or tea ☕ ) and then re-run the `GET _data_stream` request. @@ -348,6 +358,8 @@ After the ILM policy has taken effect, the original `.ds-datastream-2022.08.26-0 ], ... ``` +% TEST[skip:todo] +% TEST[continued] Run a search query on the datastream (note that when querying downsampled indices there are [a few nuances to be aware of](./downsampling-time-series-data-stream.md#querying-downsampled-indices-notes)). @@ -435,12 +447,15 @@ The new downsampled index contains just one document that includes the `min`, `m } } ``` +% TEST[skip:todo] +% TEST[continued] Use the [data stream stats API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-data-streams-stats-1) to get statistics for the data stream, including the storage size. ```console GET /_data_stream/datastream/_stats?human=true ``` +% TEST[continued] ```console-result { @@ -464,6 +479,8 @@ GET /_data_stream/datastream/_stats?human=true ] } ``` +% TEST[skip:todo] +% TEST[continued] This example demonstrates how downsampling works as part of an ILM policy to reduce the storage size of metrics data as it becomes less current and less frequently queried. diff --git a/manage-data/data-store/data-streams/set-up-data-stream.md b/manage-data/data-store/data-streams/set-up-data-stream.md index dc1c0be0ae..11fd134bd5 100644 --- a/manage-data/data-store/data-streams/set-up-data-stream.md +++ b/manage-data/data-store/data-streams/set-up-data-stream.md @@ -144,6 +144,7 @@ PUT _component_template/my-settings } } ``` +% TEST[continued] ## Create an index template [create-index-template] @@ -172,6 +173,7 @@ PUT _index_template/my-index-template } } ``` +% TEST[continued] ## Create the data stream [create-data-stream] @@ -193,12 +195,15 @@ POST my-data-stream/_doc "message": "192.0.2.42 - - [06/May/2099:16:21:15 +0000] \"GET /images/bg.jpg HTTP/1.0\" 200 24736" } ``` +% TEST[continued] You can also manually create the stream using the [create data stream API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-create-data-stream). The stream’s name must still match one of your template’s index patterns. ```console PUT _data_stream/my-data-stream ``` +% TEST[continued] +% TEST[s/my-data-stream/my-data-stream-alt/] After it's been created, you can view and manage this and other data streams from the **Stack Management > Index Management** view. Refer to [Manage a data stream](./manage-data-stream.md) for details. @@ -218,6 +223,7 @@ To convert an index alias with a write index to a data stream with the same name ```console POST _data_stream/_migrate/my-time-series-data ``` +% TEST[continued] ## Get information about a data stream [get-info-about-data-stream] @@ -229,6 +235,7 @@ You can also use the [get data stream API](https://www.elastic.co/docs/api/doc/e ```console GET _data_stream/my-data-stream ``` +% TEST[continued] ## Delete a data stream [delete-data-stream] @@ -240,4 +247,5 @@ You can also use the [delete data stream API](https://www.elastic.co/docs/api/do ```console DELETE _data_stream/my-data-stream ``` +% TEST[continued] diff --git a/manage-data/data-store/data-streams/set-up-tsds.md b/manage-data/data-store/data-streams/set-up-tsds.md index 6035713f63..6471722aac 100644 --- a/manage-data/data-store/data-streams/set-up-tsds.md +++ b/manage-data/data-store/data-streams/set-up-tsds.md @@ -150,6 +150,7 @@ PUT _index_template/my-weather-sensor-index-template } } ``` +% TEST[continued] ## Create the TSDS [create-tsds] @@ -182,12 +183,15 @@ POST metrics-weather_sensors-dev/_doc "humidity": 88.9 } ``` +% TEST[skip: The @timestamp value won’t match an accepted range in the TSDS] You can also manually create the TSDS using the [create data stream API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-create-data-stream). The TSDS’s name must still match one of your template’s index patterns. ```console PUT _data_stream/metrics-weather_sensors-dev ``` +% TEST[setup:tsds_template] +% TEST[teardown:tsds_cleanup] ## Secure the TSDS [secure-tsds] diff --git a/manage-data/data-store/data-streams/use-data-stream.md b/manage-data/data-store/data-streams/use-data-stream.md index 1b17f34f48..0ffbc39822 100644 --- a/manage-data/data-store/data-streams/use-data-stream.md +++ b/manage-data/data-store/data-streams/use-data-stream.md @@ -100,6 +100,8 @@ To re-open a closed backing index, submit an [open index API request](https://ww ```console POST /.ds-my-data-stream-2099.03.07-000001/_open/ ``` +% TEST[setup:my_index] +% TEST[s/.ds-my-data-stream-2099.03.07-000001/my-index-000001/] To re-open all closed backing indices for a data stream, submit an open index API request to the stream: @@ -124,6 +126,7 @@ POST /_reindex } } ``` +% TEST[continued] ## Update documents in a data stream by query [update-docs-in-a-data-stream-by-query] @@ -223,6 +226,10 @@ Response: } } ``` +% TESTRESPONSE[s/"took": 20/"took": $body.took/] +% TESTRESPONSE[s/"max_score": 0.2876821/"max_score": $body.hits.max_score/] +% TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.03.08-000003"/"_index": $body.hits.hits.0._index/] +% TESTRESPONSE[s/"_score": 0.2876821/"_score": $body.hits.hits.0._score/] 1. Backing index containing the matching document 2. Document ID for the document @@ -242,12 +249,19 @@ PUT /.ds-my-data-stream-2099-03-08-000003/_doc/bfspvnIBr7VVZlfp2lqX?if_seq_no=0& "message": "Login successful" } ``` +% TEST[setup:my_index] +% TEST[s/.ds-my-data-stream-2099.03.08-000003/my-index-000001/] +% TEST[s/bfspvnIBr7VVZlfp2lqX/1/] +% TEST[s/if_seq_no=0/if_seq_no=1/] To delete the document, use the [delete API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-delete): ```console DELETE /.ds-my-data-stream-2099.03.08-000003/_doc/bfspvnIBr7VVZlfp2lqX ``` +% TEST[setup:my_index] +% TEST[s/.ds-my-data-stream-2099.03.08-000003/my-index-000001/] +% TEST[s/bfspvnIBr7VVZlfp2lqX/1/] To delete or update multiple documents with a single request, use the [bulk API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-bulk)'s `delete`, `index`, and `update` actions. For `index` actions, include valid [`if_seq_no` and `if_primary_term`](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-bulk#bulk-optimistic-concurrency-control) arguments. @@ -256,4 +270,7 @@ PUT /_bulk?refresh { "index": { "_index": ".ds-my-data-stream-2099.03.08-000003", "_id": "bfspvnIBr7VVZlfp2lqX", "if_seq_no": 0, "if_primary_term": 1 } } { "@timestamp": "2099-03-08T11:06:07.000Z", "user": { "id": "8a4f500d" }, "message": "Login successful" } ``` +% TEST[setup:my_index] +% TEST[s/.ds-my-data-stream-2099.03.08-000003/my-index-000001/] +% TEST[s/bfspvnIBr7VVZlfp2lqX/1/] diff --git a/manage-data/data-store/mapping/define-runtime-fields-in-search-request.md b/manage-data/data-store/mapping/define-runtime-fields-in-search-request.md index d2c13c490e..d08529f111 100644 --- a/manage-data/data-store/mapping/define-runtime-fields-in-search-request.md +++ b/manage-data/data-store/mapping/define-runtime-fields-in-search-request.md @@ -34,6 +34,7 @@ GET my-index-000001/_search } } ``` +% TEST[continued] ## Create runtime fields that use other runtime fields [runtime-search-request-examples] @@ -71,6 +72,7 @@ PUT my-index-000001/_mapping } } ``` +% TEST[continued] Runtime fields take precedence over fields defined with the same name in the index mappings. This flexibility allows you to shadow existing fields and calculate a different value, without modifying the field itself. If you made a mistake in your index mapping, you can use runtime fields to calculate values that [override values](override-field-values-at-query-time.md) in the mapping during the search request. @@ -93,6 +95,8 @@ GET my-index-000001/_search } } ``` +% TEST[continued] +% TEST[s/_search/_search?filter_path=aggregations/] The response includes the aggregation results without changing the values for the underlying data: @@ -135,6 +139,8 @@ GET my-index-000001/_search } } ``` +% TEST[continued] +% TEST[s/_search/_search?filter_path=aggregations/] Even though the `duration` runtime field only exists in the context of a search query, you can search and aggregate on that field. This flexibility is incredibly powerful, enabling you to rectify mistakes in your index mappings and dynamically complete calculations all within a single search request. diff --git a/manage-data/data-store/mapping/dynamic-templates.md b/manage-data/data-store/mapping/dynamic-templates.md index 6d6adebd7a..8f27f0af06 100644 --- a/manage-data/data-store/mapping/dynamic-templates.md +++ b/manage-data/data-store/mapping/dynamic-templates.md @@ -35,6 +35,7 @@ Dynamic templates are specified as an array of named objects: ... ] ``` +% NOTCONSOLE 1. The template name can be any string value. 2. The match conditions can include any of : `match_mapping_type`, `match`, `match_pattern`, `unmatch`, `path_match`, `path_unmatch`. @@ -190,6 +191,7 @@ The `match_pattern` parameter adjusts the behavior of the `match` parameter to s "match_pattern": "regex", "match": "^profit_\d+$" ``` +% NOTCONSOLE The following example matches all `string` fields whose name starts with `long_` (except for those which end with `_text`) and maps them as `long` fields: @@ -355,6 +357,8 @@ PUT my-index-000001/_doc/2 } } ``` +% TEST[continued] +% TEST[catch:bad_request] ## Template variables [template-variables] @@ -481,12 +485,14 @@ PUT my-index-000001/_doc/1 "count": 5 } ``` +% TEST[continued] When you view the mapping, you’ll see that the `english` field is a runtime field of type `keyword`: ```console GET my-index-000001/_mapping ``` +% TEST[continued] ```console-result { diff --git a/manage-data/data-store/mapping/explicit-mapping.md b/manage-data/data-store/mapping/explicit-mapping.md index f49cbfb52e..a82debee29 100644 --- a/manage-data/data-store/mapping/explicit-mapping.md +++ b/manage-data/data-store/mapping/explicit-mapping.md @@ -53,6 +53,7 @@ PUT /my-index-000001/_mapping } } ``` +% TEST[continued] ## Update the mapping of a field [update-mapping] @@ -73,6 +74,7 @@ You can use the [get mapping](https://www.elastic.co/docs/api/doc/elasticsearch/ ```console GET /my-index-000001/_mapping ``` +% TEST[continued] The API returns the following response: @@ -112,6 +114,7 @@ The following request retrieves the mapping for the `employee-id` field. ```console GET /my-index-000001/_mapping/field/employee-id ``` +% TEST[continued] The API returns the following response: diff --git a/manage-data/data-store/mapping/explore-data-with-runtime-fields.md b/manage-data/data-store/mapping/explore-data-with-runtime-fields.md index 4fe70b3128..f9ec082f7f 100644 --- a/manage-data/data-store/mapping/explore-data-with-runtime-fields.md +++ b/manage-data/data-store/mapping/explore-data-with-runtime-fields.md @@ -57,12 +57,14 @@ POST /my-index-000001/_bulk?refresh {"index":{}} {"timestamp":"2020-04-30T14:31:28-05:00","message":"not a valid apache log"} ``` +% TEST[continued] At this point, you can view how {{es}} stores your raw data. ```console GET /my-index-000001 ``` +% TEST[continued] The mapping contains two fields: `@timestamp` and `message`. @@ -88,6 +90,7 @@ The mapping contains two fields: `@timestamp` and `message`. } } ``` +% TESTRESPONSE[s/.../"settings": $body.my-index-000001.settings/] ## Define a runtime field with a grok pattern [runtime-examples-grok] @@ -110,6 +113,7 @@ PUT my-index-000001/_mappings } } ``` +% TEST[continued] 1. This condition ensures that the script doesn’t crash even if the pattern of the message doesn’t match. @@ -136,6 +140,7 @@ GET my-index-000001/_search "fields" : ["http.clientip"] } ``` +% TEST[continued] ## Define a composite runtime field [runtime-examples-grok-composite] @@ -164,6 +169,7 @@ PUT my-index-000001/_mappings } } ``` +% TEST[continued] ### Search for a specific IP address [runtime-examples-grok-ip] @@ -180,6 +186,7 @@ GET my-index-000001/_search "fields" : ["*"] } ``` +% TEST[continued] The API returns the following result. Because `http` is a `composite` runtime field, the response includes each of the sub-fields under `fields`, including any associated values that match the query. Without building your data structure in advance, you can search and explore your data in meaningful ways to experiment and determine which fields to index. @@ -226,6 +233,8 @@ The API returns the following result. Because `http` is a `composite` runtime fi } } ``` +% TESTRESPONSE[s/.../"took" : $body.took,"timed_out" : $body.timed_out,"_shards" : $body._shards,/] +% TESTRESPONSE[s/"_id" : "sRVHBnwBB-qjgFni7h_O"/"_id": $body.hits.hits.0._id/] Also, remember that `if` statement in the script? @@ -252,6 +261,7 @@ GET my-index-000001/_search } } ``` +% TEST[continued] The response includes the document where the log format doesn’t match, but the timestamp falls within the defined range. @@ -287,6 +297,9 @@ The response includes the document where the log format doesn’t match, but the } } ``` +% TESTRESPONSE[s/.../"took" : $body.took,"timed_out" : $body.timed_out,"_shards" : $body._shards,/] +% TESTRESPONSE[s/"_id" : "hdEhyncBRSB6iD-PoBqe"/"_id": $body.hits.hits.0._id/] +% TESTRESPONSE[s/"_id" : "htEhyncBRSB6iD-PoBqe"/"_id": $body.hits.hits.1._id/] @@ -310,6 +323,7 @@ PUT my-index-000001/_mappings } } ``` +% TEST[continued] Similarly, you can define a dissect pattern to extract the [HTTP response code](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status): @@ -327,6 +341,7 @@ PUT my-index-000001/_mappings } } ``` +% TEST[continued] You can then run a query to retrieve a specific HTTP response using the `http.responses` runtime field. Use the `fields` parameter of the `_search` request to indicate which fields you want to retrieve: @@ -341,6 +356,7 @@ GET my-index-000001/_search "fields" : ["http.client_ip","timestamp","http.verb"] } ``` +% TEST[continued] The response includes a single document where the HTTP response is `304`: @@ -378,5 +394,7 @@ The response includes a single document where the HTTP response is `304`: } } ``` +% TESTRESPONSE[s/.../"took" : $body.took,"timed_out" : $body.timed_out,"_shards" : $body._shards,/] +% TESTRESPONSE[s/"_id" : "A2qDy3cBWRMvVAuI7F8M"/"_id": $body.hits.hits.0._id/] diff --git a/manage-data/data-store/mapping/index-runtime-field.md b/manage-data/data-store/mapping/index-runtime-field.md index c38d1528f9..1539732cdb 100644 --- a/manage-data/data-store/mapping/index-runtime-field.md +++ b/manage-data/data-store/mapping/index-runtime-field.md @@ -63,6 +63,7 @@ POST my-index-000001/_bulk?refresh=true {"index":{}} {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"} ``` +% TEST[continued] After talking to a few site engineers, you realize that the sensors should be reporting at least *double* the current values, but potentially higher. You create a runtime field named `voltage_corrected` that retrieves the current voltage and multiplies it by `2`: @@ -84,6 +85,7 @@ PUT my-index-000001/_mapping } } ``` +% TEST[continued] You retrieve the calculated values using the [`fields`](elasticsearch://reference/elasticsearch/rest-apis/retrieve-selected-fields.md) parameter on the `_search` API: @@ -97,6 +99,8 @@ GET my-index-000001/_search "size": 2 } ``` +% TEST[continued] +% TEST[s/_search/_search?filter_path=hits/] After reviewing the sensor data and running some tests, you determine that the multiplier for reported sensor data should be `4`. To gain greater performance, you decide to index the `voltage_corrected` runtime field with the new `multiplier` parameter. @@ -156,6 +160,7 @@ POST my-index-000001/_bulk?refresh=true { "index": {}} { "timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"} ``` +% TEST[continued] You can now retrieve calculated values in a search query, and find documents based on precise values. The following range query returns all documents where the calculated `voltage_corrected` is greater than or equal to `16`, but less than or equal to `20`. Again, use the [`fields`](elasticsearch://reference/elasticsearch/rest-apis/retrieve-selected-fields.md) parameter on the `_search` API to retrieve the fields you want: @@ -174,6 +179,8 @@ POST my-index-000001/_search "fields": ["voltage_corrected", "node"] } ``` +% TEST[continued] +% TEST[s/_search/_search?filter_path=hits/] The response includes the `voltage_corrected` field for the documents that match the range query, based on the calculated value of the included script: @@ -228,4 +235,6 @@ The response includes the `voltage_corrected` field for the documents that match } } ``` +% TESTRESPONSE[s/"_id" : "yoSLrHgBdg9xpPrUZz_P"/"_id": $body.hits.hits.0._id/] +% TESTRESPONSE[s/"_id" : "y4SLrHgBdg9xpPrUZz_P"/"_id": $body.hits.hits.1._id/] diff --git a/manage-data/data-store/mapping/map-runtime-field.md b/manage-data/data-store/mapping/map-runtime-field.md index 9d56e5418f..592b59d95e 100644 --- a/manage-data/data-store/mapping/map-runtime-field.md +++ b/manage-data/data-store/mapping/map-runtime-field.md @@ -115,6 +115,7 @@ PUT my-index-000001/_mapping } } ``` +% TEST[continued] :::::{admonition} Downstream impacts Updating or removing a runtime field while a dependent query is running can return inconsistent results. Each shard might have access to different versions of the script, depending on when the mapping change takes effect. diff --git a/manage-data/data-store/mapping/override-field-values-at-query-time.md b/manage-data/data-store/mapping/override-field-values-at-query-time.md index f703548aaf..c88dfafe22 100644 --- a/manage-data/data-store/mapping/override-field-values-at-query-time.md +++ b/manage-data/data-store/mapping/override-field-values-at-query-time.md @@ -42,6 +42,7 @@ GET my-index-000001/_search } } ``` +% TEST[continued] The response includes indexed values for documents matching model number `HG537PU`: @@ -83,6 +84,9 @@ The response includes indexed values for documents matching model number `HG537P } } ``` +% TESTRESPONSE[s/.../"took" : $body.took,"timed_out" : $body.timed_out,"_shards" : $body._shards,/] +% TESTRESPONSE[s/"_id" : "F1BeSXYBg_szTodcYCmk"/"_id": $body.hits.hits.0._id/] +% TESTRESPONSE[s/"_id" : "l02aSXYBkpNf6QRDO62Q"/"_id": $body.hits.hits.1._id/] The following request defines a runtime field where the script evaluates the `model_number` field where the value is `HG537PU`. For each match, the script multiplies the value for the `voltage` field by `1.7`. @@ -110,6 +114,7 @@ POST my-index-000001/_search "fields": ["measures.voltage"] } ``` +% TEST[continued] Looking at the response, the calculated values for `measures.voltage` on each result are `7.14` and `6.8`. That’s more like it! The runtime field calculated this value as part of the search request without modifying the mapped value, which still returns in the response: @@ -161,4 +166,7 @@ Looking at the response, the calculated values for `measures.voltage` on each re } } ``` +% TESTRESPONSE[s/.../"took" : $body.took,"timed_out" : $body.timed_out,"_shards" : $body._shards,/] +% TESTRESPONSE[s/"_id" : "F1BeSXYBg_szTodcYCmk"/"_id": $body.hits.hits.0._id/] +% TESTRESPONSE[s/"_id" : "l02aSXYBkpNf6QRDO62Q"/"_id": $body.hits.hits.1._id/] diff --git a/manage-data/data-store/mapping/retrieve-runtime-field.md b/manage-data/data-store/mapping/retrieve-runtime-field.md index a6da515e0f..f84726eaa5 100644 --- a/manage-data/data-store/mapping/retrieve-runtime-field.md +++ b/manage-data/data-store/mapping/retrieve-runtime-field.md @@ -64,6 +64,7 @@ POST /my-index-000001/_bulk?refresh { "index": {}} { "@timestamp": "2020-04-30T14:31:43-05:00", "message" : "247.37.0.0 - - [2020-04-30T14:31:43-05:00] \"GET /french/images/nav_venue_off.gif HTTP/1.0\" 304 0"} ``` +% TEST[continued] ## Search for the calculated day of week [runtime-search-dayofweek] @@ -80,6 +81,7 @@ GET my-index-000001/_search "_source": false } ``` +% TEST[continued] The previous request returns the `day_of_week` field for all matching documents. We can define another runtime field called `client_ip` that also operates on the `message` field and will further refine the query: @@ -96,6 +98,7 @@ PUT /my-index-000001/_mapping } } ``` +% TEST[continued] Run another query, but search for a specific IP address using the `client_ip` runtime field: @@ -111,6 +114,7 @@ GET my-index-000001/_search "fields" : ["*"] } ``` +% TEST[continued] This time, the response includes only two hits. The value for `day_of_week` (`Sunday`) was calculated at query time using the runtime script defined in the mapping, and the result includes only documents matching the `211.11.9.0` IP address. @@ -151,6 +155,9 @@ This time, the response includes only two hits. The value for `day_of_week` (`Su } } ``` +% TESTRESPONSE[s/.../"took" : $body.took,"timed_out" : $body.timed_out,"_shards" : $body._shards,/] +% TESTRESPONSE[s/"_id" : "oWs5KXYB-XyJbifr9mrz"/"_id": $body.hits.hits.0._id/] +% TESTRESPONSE[s/"day_of_week" : [\n\s+"Sunday"\n\s]/"day_of_week": $body.hits.hits.0.fields.day_of_week/] ## Retrieve fields from related indices [lookup-runtime-fields] @@ -256,6 +263,7 @@ The above search returns the country and city from the `ip_location` index for e } } ``` +% TESTRESPONSE[s/"took": 3/"took": $body.took/] The response of lookup fields are grouped to maintain the independence of each document from the lookup index. The lookup query for each input value is expected to match at most one document on the lookup index. If the lookup query matches more than one documents, then a random document will be selected. diff --git a/manage-data/data-store/text-analysis/specify-an-analyzer.md b/manage-data/data-store/text-analysis/specify-an-analyzer.md index 44553bf8d4..d47dcf730c 100644 --- a/manage-data/data-store/text-analysis/specify-an-analyzer.md +++ b/manage-data/data-store/text-analysis/specify-an-analyzer.md @@ -119,6 +119,7 @@ GET my-index-000001/_search } } ``` +% TEST[s/^/PUT my-index-000001\n/] ## Specify the search analyzer for a field [specify-search-field-analyzer] diff --git a/manage-data/ingest/transform-enrich/example-enrich-data-based-on-exact-values.md b/manage-data/ingest/transform-enrich/example-enrich-data-based-on-exact-values.md index b10658a74a..40c96027e0 100644 --- a/manage-data/ingest/transform-enrich/example-enrich-data-based-on-exact-values.md +++ b/manage-data/ingest/transform-enrich/example-enrich-data-based-on-exact-values.md @@ -46,12 +46,15 @@ PUT /_enrich/policy/users-policy } } ``` +% TEST[continued] Use the [execute enrich policy API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-enrich-execute-policy) to create an enrich index for the policy. ```console POST /_enrich/policy/users-policy/_execute?wait_for_completion=false ``` +% TEST[s/?wait_for_completion=false//] +% TEST[continued] Use the [create or update pipeline API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-ingest-put-pipeline) to create an ingest pipeline. In the pipeline, add an [enrich processor](elasticsearch://reference/ingestion-tools/enrich-processor/enrich-processor.md) that includes: @@ -75,6 +78,7 @@ PUT /_ingest/pipeline/user_lookup ] } ``` +% TEST[continued] Use the ingest pipeline to index a document. The incoming document should include the `field` specified in your enrich processor. @@ -84,12 +88,14 @@ PUT /my-index-000001/_doc/my_id?pipeline=user_lookup "email": "mardy.brown@asciidocsmith.com" } ``` +% TEST[continued] To verify the enrich processor matched and appended the appropriate field data, use the [get API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-get) to view the indexed document. ```console GET /my-index-000001/_doc/my_id ``` +% TEST[continued] The API returns the following response: @@ -114,4 +120,5 @@ The API returns the following response: } } ``` +% TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term":1/"_primary_term" : $body._primary_term/] diff --git a/manage-data/ingest/transform-enrich/example-enrich-data-based-on-geolocation.md b/manage-data/ingest/transform-enrich/example-enrich-data-based-on-geolocation.md index d386a2673d..757fb3feac 100644 --- a/manage-data/ingest/transform-enrich/example-enrich-data-based-on-geolocation.md +++ b/manage-data/ingest/transform-enrich/example-enrich-data-based-on-geolocation.md @@ -42,6 +42,7 @@ PUT /postal_codes/_doc/1?refresh=wait_for "postal_code": "96598" } ``` +% TEST[continued] Use the [create enrich policy API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-enrich-put-policy) to create an enrich policy with the `geo_match` policy type. This policy must include: @@ -59,12 +60,15 @@ PUT /_enrich/policy/postal_policy } } ``` +% TEST[continued] Use the [execute enrich policy API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-enrich-execute-policy) to create an enrich index for the policy. ```console POST /_enrich/policy/postal_policy/_execute?wait_for_completion=false ``` +% TEST[s/?wait_for_completion=false//] +% TEST[continued] Use the [create or update pipeline API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-ingest-put-pipeline) to create an ingest pipeline. In the pipeline, add an [enrich processor](elasticsearch://reference/ingestion-tools/enrich-processor/enrich-processor.md) that includes: @@ -89,6 +93,7 @@ PUT /_ingest/pipeline/postal_lookup ] } ``` +% TEST[continued] Use the ingest pipeline to index a document. The incoming document should include the `field` specified in your enrich processor. @@ -100,12 +105,14 @@ PUT /users/_doc/0?pipeline=postal_lookup "geo_location": "POINT (13.5 52.5)" } ``` +% TEST[continued] To verify the enrich processor matched and appended the appropriate field data, use the [get API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-get) to view the indexed document. ```console GET /users/_doc/0 ``` +% TEST[continued] The API returns the following response: @@ -131,4 +138,5 @@ The API returns the following response: } } ``` +% TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term":1/"_primary_term" : $body._primary_term/] diff --git a/manage-data/ingest/transform-enrich/example-enrich-data-by-matching-value-to-range.md b/manage-data/ingest/transform-enrich/example-enrich-data-by-matching-value-to-range.md index 1098ba802a..a848cdb8aa 100644 --- a/manage-data/ingest/transform-enrich/example-enrich-data-by-matching-value-to-range.md +++ b/manage-data/ingest/transform-enrich/example-enrich-data-by-matching-value-to-range.md @@ -37,6 +37,7 @@ PUT /networks/_doc/1?refresh=wait_for "department": "OPS" } ``` +% TEST[continued] Use the create enrich policy API to create an enrich policy with the `range` policy type. This policy must include: @@ -56,12 +57,15 @@ PUT /_enrich/policy/networks-policy } } ``` +% TEST[continued] Use the [execute enrich policy API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-enrich-execute-policy) to create an enrich index for the policy. ```console POST /_enrich/policy/networks-policy/_execute?wait_for_completion=false ``` +% TEST[s/?wait_for_completion=false//] +% TEST[continued] Use the [create or update pipeline API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-ingest-put-pipeline) to create an ingest pipeline. In the pipeline, add an [enrich processor](elasticsearch://reference/ingestion-tools/enrich-processor/enrich-processor.md) that includes: @@ -85,6 +89,7 @@ PUT /_ingest/pipeline/networks_lookup ] } ``` +% TEST[continued] Use the ingest pipeline to index a document. The incoming document should include the `field` specified in your enrich processor. @@ -94,12 +99,14 @@ PUT /my-index-000001/_doc/my_id?pipeline=networks_lookup "ip": "10.100.34.1" } ``` +% TEST[continued] To verify the enrich processor matched and appended the appropriate field data, use the [get API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-get) to view the indexed document. ```console GET /my-index-000001/_doc/my_id ``` +% TEST[continued] The API returns the following response: @@ -123,4 +130,5 @@ The API returns the following response: } } ``` +% TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term":1/"_primary_term" : $body._primary_term/] diff --git a/manage-data/ingest/transform-enrich/example-parse-logs.md b/manage-data/ingest/transform-enrich/example-parse-logs.md index d38317bcee..b465eb9dd7 100644 --- a/manage-data/ingest/transform-enrich/example-parse-logs.md +++ b/manage-data/ingest/transform-enrich/example-parse-logs.md @@ -19,6 +19,7 @@ The logs you want to parse look similar to this: ```txt 212.87.37.154 - - [05/May/2099:16:21:15 +0000] "GET /favicon.ico HTTP/1.1" 200 3638 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36" ``` +% NOTCONSOLE These logs contain a timestamp, IP address, and user agent. You want to give these three items their own field in {{es}} for faster searches and visualizations. You also want to know where the request is coming from. diff --git a/manage-data/ingest/transform-enrich/ingest-pipelines.md b/manage-data/ingest/transform-enrich/ingest-pipelines.md index ed94272efd..b66bfdb6ee 100644 --- a/manage-data/ingest/transform-enrich/ingest-pipelines.md +++ b/manage-data/ingest/transform-enrich/ingest-pipelines.md @@ -78,6 +78,7 @@ PUT _ingest/pipeline/my-pipeline ] } ``` +% TESTSETUP ## Manage pipeline versions [manage-pipeline-versions] @@ -91,6 +92,7 @@ PUT _ingest/pipeline/my-pipeline-id "processors": [ ... ] } ``` +% TEST[s/.../{"lowercase": {"field":"my-keyword-field"}}/] To unset the `version` number using the API, replace or update the pipeline without specifying the `version` parameter. @@ -187,6 +189,8 @@ The API returns transformed documents: ] } ``` +% TESTRESPONSE[s/"2099-03-07T11:04:03.000Z"/$body.docs.0.doc._ingest.timestamp/] +% TESTRESPONSE[s/"2099-03-07T11:04:04.000Z"/$body.docs.1.doc._ingest.timestamp/] ## Add a pipeline to an indexing request [add-pipeline-to-indexing-request] @@ -206,6 +210,8 @@ PUT my-data-stream/_bulk?pipeline=my-pipeline { "create":{ } } { "@timestamp": "2099-03-07T11:04:07.000Z", "my-keyword-field": "bar" } ``` +% TEST[setup:my_data_stream] +% TEST[teardown:data_stream_cleanup] You can also use the `pipeline` parameter with the [update by query](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-update-by-query) or [reindex](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-reindex) APIs. @@ -224,6 +230,8 @@ POST _reindex } } ``` +% TEST[setup:my_data_stream] +% TEST[teardown:data_stream_cleanup] ## Set a default pipeline [set-default-pipeline] @@ -612,6 +620,7 @@ PUT _ingest/pipeline/my-pipeline ] } ``` +% TEST[s/.../{"lowercase": {"field":"my-keyword-field"}}/] Additional information about the pipeline failure may be available in the document metadata fields `on_failure_message`, `on_failure_processor_type`, `on_failure_processor_tag`, and `on_failure_pipeline`. These fields are accessible only from within an `on_failure` block. @@ -632,6 +641,7 @@ PUT _ingest/pipeline/my-pipeline ] } ``` +% TEST[s/.../{"lowercase": {"field":"my-keyword-field"}}/] ## Conditionally run a processor [conditionally-run-processor] diff --git a/manage-data/lifecycle/data-stream/tutorial-create-data-stream-with-lifecycle.md b/manage-data/lifecycle/data-stream/tutorial-create-data-stream-with-lifecycle.md index 7bde9d231e..590d27cdcb 100644 --- a/manage-data/lifecycle/data-stream/tutorial-create-data-stream-with-lifecycle.md +++ b/manage-data/lifecycle/data-stream/tutorial-create-data-stream-with-lifecycle.md @@ -72,6 +72,7 @@ You can use the [get data stream lifecycle API](https://www.elastic.co/docs/api/ ```console GET _data_stream/my-data-stream/_lifecycle ``` +% TEST[continued] The result will look like this: @@ -103,6 +104,7 @@ If you want to see more information about how the data stream lifecycle is appli ```console GET .ds-my-data-stream-*/_lifecycle/explain ``` +% TEST[continued] The result will look like this: @@ -122,6 +124,7 @@ The result will look like this: } } ``` +% TESTRESPONSE[skip:the result is for illustrating purposes only] 1. The name of the backing index. 2. If it is managed by the built-in data stream lifecycle. diff --git a/manage-data/lifecycle/data-stream/tutorial-data-stream-retention.md b/manage-data/lifecycle/data-stream/tutorial-data-stream-retention.md index 8c0487ca3d..e5fb3a21db 100644 --- a/manage-data/lifecycle/data-stream/tutorial-data-stream-retention.md +++ b/manage-data/lifecycle/data-stream/tutorial-data-stream-retention.md @@ -20,6 +20,7 @@ You can verify if a data steam is managed by the data stream lifecycle via the [ ```console GET _data_stream/my-data-stream/_lifecycle ``` +% TEST[continued] The result should look like this: @@ -35,6 +36,7 @@ The result should look like this: ] } ``` +% TESTRESPONSE[skip:the result is for illustrating purposes only] 1. The name of your data stream. 2. Ensure that the lifecycle is enabled, meaning this should be `true`. diff --git a/manage-data/lifecycle/data-stream/tutorial-migrate-ilm-managed-data-stream-to-data-stream-lifecycle.md b/manage-data/lifecycle/data-stream/tutorial-migrate-ilm-managed-data-stream-to-data-stream-lifecycle.md index 0c64035a6e..580a1ed3b4 100644 --- a/manage-data/lifecycle/data-stream/tutorial-migrate-ilm-managed-data-stream-to-data-stream-lifecycle.md +++ b/manage-data/lifecycle/data-stream/tutorial-migrate-ilm-managed-data-stream-to-data-stream-lifecycle.md @@ -63,6 +63,7 @@ PUT _index_template/dsl-data-stream-template } } ``` +% TEST[continued] We’ll now index a document targetting `dsl-data-stream` to create the data stream and we’ll also manually rollover the data stream to have another generation index created: @@ -73,16 +74,23 @@ POST dsl-data-stream/_doc? "message": "192.0.2.42 - - [06/May/2099:16:21:15 +0000] \"GET /images/bg.jpg HTTP/1.0\" 200 24736" } ``` +% TEST[continued] ```console POST dsl-data-stream/_rollover ``` +% TEST[continued] +% TEST[continued] We’ll use the [GET _data_stream](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-get-data-stream) API to inspect the state of the data stream: ```console GET _data_stream/dsl-data-stream ``` +% TEST[continued] +% TEST[continued] +% TEST[continued] +% TEST[continued] Inspecting the response we’ll see that both backing indices are managed by {{ilm-init}} and that the next generation index will also be managed by {{ilm-init}}: @@ -125,6 +133,11 @@ Inspecting the response we’ll see that both backing indices are managed by {{i ] } ``` +% TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000001"/"index_name": $body.data_streams.0.indices.0.index_name/] +% TESTRESPONSE[s/"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg"/"index_uuid": $body.data_streams.0.indices.0.index_uuid/] +% TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000002"/"index_name": $body.data_streams.0.indices.1.index_name/] +% TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/] +% TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_store":{"enabled": false, "indices": [], "rollover_on_write": true}/] 1. The name of the backing index. 2. For each backing index we display the value of the [prefer_ilm](elasticsearch://reference/elasticsearch/configuration-reference/data-stream-lifecycle-settings.md#index-lifecycle-prefer-ilm) configuration which will indicate if {{ilm-init}} takes precedence over data stream lifecycle in case both systems are configured for an index. @@ -168,6 +181,7 @@ PUT _index_template/dsl-data-stream-template } } ``` +% TEST[continued] 1. The `prefer_ilm` setting will now be configured on the **new** backing indices (created by rolling over the data stream) such that {{ilm-init}} does *not* take precedence over data stream lifecycle. 2. We’re configuring the data stream lifecycle so *new* data streams will be managed by data stream lifecycle. @@ -183,6 +197,7 @@ PUT _data_stream/dsl-data-stream/_lifecycle "data_retention": "7d" } ``` +% TEST[continued] We can inspect the data stream to check that the next generation will indeed be managed by data stream lifecycle: @@ -235,6 +250,11 @@ GET _data_stream/dsl-data-stream ] } ``` +% TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000001"/"index_name": $body.data_streams.0.indices.0.index_name/] +% TESTRESPONSE[s/"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg"/"index_uuid": $body.data_streams.0.indices.0.index_uuid/] +% TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000002"/"index_name": $body.data_streams.0.indices.1.index_name/] +% TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/] +% TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_store":{"enabled": false, "indices": [], "rollover_on_write": true}/] 1. The existing backing index will continue to be managed by {{ilm-init}} 2. The existing backing index will continue to be managed by {{ilm-init}} @@ -304,6 +324,13 @@ GET _data_stream/dsl-data-stream ] } ``` +% TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000001"/"index_name": $body.data_streams.0.indices.0.index_name/] +% TESTRESPONSE[s/"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg"/"index_uuid": $body.data_streams.0.indices.0.index_uuid/] +% TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000002"/"index_name": $body.data_streams.0.indices.1.index_name/] +% TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/] +% TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000003"/"index_name": $body.data_streams.0.indices.2.index_name/] +% TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8abcd1"/"index_uuid": $body.data_streams.0.indices.2.index_uuid/] +% TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_store":{"enabled": false, "indices": [], "rollover_on_write": true}/] 1. The backing indices that existed before rollover will continue to be managed by {{ilm-init}} 2. The backing indices that existed before rollover will continue to be managed by {{ilm-init}} @@ -330,6 +357,7 @@ PUT _data_stream/dsl-data-stream/_lifecycle "enabled": false <1> } ``` +% TEST[continued] 1. The `enabled` flag can be ommitted and defaults to `true` however, here we explicitly configure it to `false` Let’s check the state of the data stream: @@ -388,6 +416,13 @@ GET _data_stream/dsl-data-stream ] } ``` +% TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000001"/"index_name": $body.data_streams.0.indices.0.index_name/] +% TESTRESPONSE[s/"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg"/"index_uuid": $body.data_streams.0.indices.0.index_uuid/] +% TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000002"/"index_name": $body.data_streams.0.indices.1.index_name/] +% TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/] +% TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000003"/"index_name": $body.data_streams.0.indices.2.index_name/] +% TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8abcd1"/"index_uuid": $body.data_streams.0.indices.2.index_uuid/] +% TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_store":{"enabled": false, "indices": [], "rollover_on_write": true}/] 1. The write index is now managed by {{ilm-init}} 2. The `lifecycle` configured on the data stream is now disabled. diff --git a/manage-data/lifecycle/data-stream/tutorial-update-existing-data-stream.md b/manage-data/lifecycle/data-stream/tutorial-update-existing-data-stream.md index 85a4eebb38..7589edce6f 100644 --- a/manage-data/lifecycle/data-stream/tutorial-update-existing-data-stream.md +++ b/manage-data/lifecycle/data-stream/tutorial-update-existing-data-stream.md @@ -44,6 +44,9 @@ The changes in the lifecycle are applied on all backing indices of the data stre ```console GET .ds-my-data-stream-*/_lifecycle/explain ``` +% TEST[continued] +% TEST[teardown:data_stream_cleanup] +% TEST[continued] The response will look like: @@ -76,6 +79,8 @@ The response will look like: } } ``` +% TEST[continued] +% TESTRESPONSE[skip:the result is for illustrating purposes only] 1. The name of the backing index. 2. This index is managed by a data stream lifecycle. @@ -97,6 +102,7 @@ To remove the lifecycle of a data stream you can use the [delete lifecycle API]( ```console DELETE _data_stream/my-data-stream/_lifecycle ``` +% TEST[continued] You can then use the [explain API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-explain-data-lifecycle) again to see that the indices are no longer managed. @@ -118,6 +124,7 @@ GET .ds-my-data-stream-*/_lifecycle/explain } } ``` +% TESTRESPONSE[skip:the result is for illustrating purposes only] 1. The name of the backing index. 2. Indication that the index is not managed by the data stream lifecycle. diff --git a/manage-data/lifecycle/index-lifecycle-management/configure-lifecycle-policy.md b/manage-data/lifecycle/index-lifecycle-management/configure-lifecycle-policy.md index 2e822e0692..8c74f4c064 100644 --- a/manage-data/lifecycle/index-lifecycle-management/configure-lifecycle-policy.md +++ b/manage-data/lifecycle/index-lifecycle-management/configure-lifecycle-policy.md @@ -188,6 +188,7 @@ PUT mylogs-pre-ilm*/_settings <1> } } ``` +% TEST[continued] 1. Updates all indices with names that start with `mylogs-pre-ilm` diff --git a/manage-data/lifecycle/index-lifecycle-management/migrate-index-allocation-filters-to-node-roles.md b/manage-data/lifecycle/index-lifecycle-management/migrate-index-allocation-filters-to-node-roles.md index 58ffd8473e..32387c7018 100644 --- a/manage-data/lifecycle/index-lifecycle-management/migrate-index-allocation-filters-to-node-roles.md +++ b/manage-data/lifecycle/index-lifecycle-management/migrate-index-allocation-filters-to-node-roles.md @@ -90,6 +90,7 @@ On {{ech}} deployments, remove the `cloud-hot-warm-allocation-0` index template ```console DELETE _template/.cloud-hot-warm-allocation-0 ``` +% TEST[skip:no cloud template] If you’re using a custom index template, update it to remove the [attribute-based allocation filters](../../../deploy-manage/distributed-architecture/shard-allocation-relocation-recovery/index-level-shard-allocation.md) you used to assign new indices to the hot tier. @@ -114,6 +115,7 @@ PUT my-index/_settings "index.routing.allocation.include._tier_preference": "data_hot" } ``` +% TEST[continued] For indices that have already transitioned out of the hot phase, the tier preference should include the appropriate fallback tiers to ensure index shards can be allocated if the preferred tier is unavailable. For example, specify the hot tier as the fallback for indices already in the warm phase. @@ -124,6 +126,8 @@ PUT my-index/_settings "index.routing.allocation.include._tier_preference": "data_warm,data_hot" } ``` +% TEST[continued] +% TEST[continued] If an index is already in the cold phase, include the cold, warm, and hot tiers. diff --git a/manage-data/lifecycle/index-lifecycle-management/start-stop-index-lifecycle-management.md b/manage-data/lifecycle/index-lifecycle-management/start-stop-index-lifecycle-management.md index 33f0e30855..03ebccf942 100644 --- a/manage-data/lifecycle/index-lifecycle-management/start-stop-index-lifecycle-management.md +++ b/manage-data/lifecycle/index-lifecycle-management/start-stop-index-lifecycle-management.md @@ -43,6 +43,7 @@ To stop the {{ilm-init}} service and pause execution of all lifecycle policies, ```console POST _ilm/stop ``` +% TEST[continued] {{ilm-init}} service runs all policies to a point where it is safe to stop. While the {{ilm-init}} service is shutting down, the status API shows {{ilm-init}} is in the `STOPPING` mode: @@ -51,6 +52,7 @@ POST _ilm/stop "operation_mode": "STOPPING" } ``` +% TESTRESPONSE[s/"STOPPING"/$body.operation_mode/] Once all policies are at a safe stopping point, {{ilm-init}} moves into the `STOPPED` mode: @@ -59,6 +61,7 @@ Once all policies are at a safe stopping point, {{ilm-init}} moves into the `STO "operation_mode": "STOPPED" } ``` +% TESTRESPONSE[s/"STOPPED"/$body.operation_mode/] ## Start {{ilm-init}} [_start_ilm_init] @@ -68,4 +71,5 @@ To restart {{ilm-init}} and resume executing policies, use the [Start API](https ```console POST _ilm/start ``` +% TEST[continued] diff --git a/manage-data/lifecycle/index-lifecycle-management/tutorial-automate-rollover.md b/manage-data/lifecycle/index-lifecycle-management/tutorial-automate-rollover.md index 625837f688..db4395d8f8 100644 --- a/manage-data/lifecycle/index-lifecycle-management/tutorial-automate-rollover.md +++ b/manage-data/lifecycle/index-lifecycle-management/tutorial-automate-rollover.md @@ -118,6 +118,7 @@ PUT _index_template/timeseries_template } } ``` +% TEST[continued] 1. Apply the template when a document is indexed into the `timeseries` target. 2. The name of the {{ilm-init}} policy used to manage the data stream. @@ -140,6 +141,7 @@ POST timeseries/_doc "@timestamp": "1591890611" } ``` +% TEST[continued] When a rollover condition in the lifecycle policy is met, the `rollover` action: @@ -162,6 +164,7 @@ For example, the following request gets information about the `timeseries` data ```console GET .ds-timeseries-*/_ilm/explain ``` +% TEST[continued] The following response shows the data stream’s first generation backing index is waiting for the `hot` phase’s `rollover` action. It remains in this state and {{ilm-init}} continues to call `check-rollover-ready` until a rollover condition is met. @@ -200,6 +203,7 @@ The following response shows the data stream’s first generation backing index } } ``` +% TESTRESPONSE[skip:no way to know if we will get this response immediately] 1. The age of the index used for calculating when to rollover the index via the `max_age` 2. The policy used to manage the index @@ -254,6 +258,7 @@ PUT _index_template/timeseries_template } } ``` +% TEST[continued] 1. Apply the template to a new index if its name starts with `timeseries-`. 2. The name of the lifecycle policy to apply to each new index. @@ -277,6 +282,7 @@ PUT timeseries-000001 } } ``` +% TEST[continued] When the rollover conditions are met, the `rollover` action: @@ -293,4 +299,5 @@ Retrieving the status information for managed indices is very similar to the dat ```console GET timeseries-*/_ilm/explain ``` +% TEST[continued] diff --git a/manage-data/lifecycle/rollup/getting-started-api.md b/manage-data/lifecycle/rollup/getting-started-api.md index 10d2736f5a..529f064049 100644 --- a/manage-data/lifecycle/rollup/getting-started-api.md +++ b/manage-data/lifecycle/rollup/getting-started-api.md @@ -29,6 +29,7 @@ Imagine you have a series of daily indices that hold sensor data (`sensor-2017-0 "node": "a" } ``` +% NOTCONSOLE ## Creating a rollup job [_creating_a_rollup_job] @@ -63,6 +64,8 @@ PUT _rollup/job/sensor ] } ``` +% TEST[setup:sensor_index] +% TEST[warning:The rollup functionality will be removed in Elasticsearch 10.0. See docs for more information.] We give the job the ID of "sensor" (in the url: `PUT _rollup/job/sensor`), and tell it to rollup the index pattern `"sensor-*"`. This job will find and rollup any index that matches that pattern. Rollup summaries are then stored in the `"sensor_rollup"` index. @@ -114,6 +117,8 @@ To start the job, execute this command: ```console POST _rollup/job/sensor/_start ``` +% TEST[setup:sensor_rollup_job] +% TEST[warning:The rollup functionality will be removed in Elasticsearch 10.0. See docs for more information.] ## Searching the rolled results [_searching_the_rolled_results] @@ -135,6 +140,8 @@ GET /sensor_rollup/_rollup_search } } ``` +% TEST[setup:sensor_prefab_data] +% TEST[warning:The rollup functionality will be removed in Elasticsearch 10.0. See docs for more information.] It’s a simple aggregation that calculates the maximum of the `temperature` field. But you’ll notice that it is being sent to the `sensor_rollup` index instead of the raw `sensor-*` indices. And you’ll also notice that it is using the `_rollup_search` endpoint. Otherwise the syntax is exactly as you’d expect. @@ -161,6 +168,9 @@ If you were to execute that query, you’d receive a result that looks like a no } } ``` +% TESTRESPONSE[s/"took" : 102/"took" : $body.$_path/] +% TESTRESPONSE[s/"_shards" : ... /"_shards" : $body.$_path/] +% TEST[warning:The rollup functionality will be removed in Elasticsearch 10.0. See docs for more information.] The only notable difference is that Rollup search results have zero `hits`, because we aren’t really searching the original, live data any more. Otherwise it’s identical syntax. @@ -201,6 +211,8 @@ GET /sensor_rollup/_rollup_search } } ``` +% TEST[setup:sensor_prefab_data] +% TEST[warning:The rollup functionality will be removed in Elasticsearch 10.0. See docs for more information.] Which returns a corresponding response: @@ -267,6 +279,8 @@ Which returns a corresponding response: } } ``` +% TESTRESPONSE[s/"took" : 93/"took" : $body.$_path/] +% TESTRESPONSE[s/"_shards" : ... /"_shards" : $body.$_path/] In addition to being more complicated (date histogram and a terms aggregation, plus an additional average metric), you’ll notice the date_histogram uses a `7d` interval instead of `60m`. diff --git a/manage-data/lifecycle/rollup/migrating-from-rollup-to-downsampling.md b/manage-data/lifecycle/rollup/migrating-from-rollup-to-downsampling.md index c5110db4b8..6a6f3ee81d 100644 --- a/manage-data/lifecycle/rollup/migrating-from-rollup-to-downsampling.md +++ b/manage-data/lifecycle/rollup/migrating-from-rollup-to-downsampling.md @@ -94,6 +94,7 @@ PUT _index_template/sensor-template } } ``` +% TEST[continued] The downsample configuration is included in the above template for a [time series data stream (TSDS)](../../data-store/data-streams/time-series-data-stream-tsds.md). Only the `downsampling` part is necessary to enable downsampling, which indicates when to downsample to what fixed interval. diff --git a/manage-data/lifecycle/rollup/rollup-search-limitations.md b/manage-data/lifecycle/rollup/rollup-search-limitations.md index 25ff85c37e..3415912a37 100644 --- a/manage-data/lifecycle/rollup/rollup-search-limitations.md +++ b/manage-data/lifecycle/rollup/rollup-search-limitations.md @@ -53,6 +53,9 @@ GET sensor_rollup/_rollup_search } } ``` +% TEST[setup:sensor_prefab_data] +% TEST[catch:/illegal_argument_exception/] +% TEST[warning:The rollup functionality will be removed in Elasticsearch 10.0. See docs for more information.] The response will tell you that the field and aggregation were not possible, because no rollup jobs were found which contained them: @@ -73,6 +76,7 @@ The response will tell you that the field and aggregation were not possible, bec "status": 400 } ``` +% TESTRESPONSE[s/"stack_trace": .../"stack_trace": $body.$_path/] ## Interval granularity [_interval_granularity] diff --git a/manage-data/lifecycle/rollup/understanding-groups.md b/manage-data/lifecycle/rollup/understanding-groups.md index 51dbf639de..f8bfb86e66 100644 --- a/manage-data/lifecycle/rollup/understanding-groups.md +++ b/manage-data/lifecycle/rollup/understanding-groups.md @@ -37,6 +37,7 @@ Rather than force the admin to decide ahead of time which individual tuples shou } } ``` +% NOTCONSOLE Allows `date_histogram` to be used on the `"timestamp"` field, `terms` aggregations to be used on the `"hostname"` and `"datacenter"` fields, and `histograms` to be used on any of `"load"`, `"net_in"`, `"net_out"` fields. @@ -59,6 +60,7 @@ Importantly, these aggs/fields can be used in any combination. This aggregation: } } ``` +% NOTCONSOLE is just as valid as this aggregation: @@ -94,6 +96,7 @@ is just as valid as this aggregation: } } ``` +% NOTCONSOLE You’ll notice that the second aggregation is not only substantially larger, it also swapped the position of the terms aggregation on `"hostname"`, illustrating how the order of aggregations does not matter to rollups. Similarly, while the `date_histogram` is required for rolling up data, it isn’t required while querying (although often used). For example, this is a valid aggregation for Rollup Search to execute: @@ -106,6 +109,7 @@ You’ll notice that the second aggregation is not only substantially larger, it } } ``` +% NOTCONSOLE Ultimately, when configuring `groups` for a job, think in terms of how you might wish to partition data in a query at a future date…​ then include those in the config. Because Rollup Search allows any order or combination of the grouped fields, you just need to decide if a field is useful for aggregating later, and how you might wish to use it (terms, histogram, etc). @@ -152,6 +156,7 @@ As an example, if your index has two types of documents: "node": "a" } ``` +% NOTCONSOLE and @@ -162,6 +167,7 @@ and "title": "Foo" } ``` +% NOTCONSOLE the best practice is to combine them into a single rollup job which covers both of these document types, like this: @@ -194,6 +200,7 @@ PUT _rollup/job/combined ] } ``` +% NOTCONSOLE ## Doc counts and overlapping jobs [_doc_counts_and_overlapping_jobs] diff --git a/manage-data/use-case-use-elasticsearch-to-manage-time-series-data.md b/manage-data/use-case-use-elasticsearch-to-manage-time-series-data.md index c43b4185cb..4b27b7e89a 100644 --- a/manage-data/use-case-use-elasticsearch-to-manage-time-series-data.md +++ b/manage-data/use-case-use-elasticsearch-to-manage-time-series-data.md @@ -301,6 +301,7 @@ PUT _component_template/my-settings } } ``` +% TEST[continued] ## Create an index template [create-ts-index-template] @@ -329,6 +330,7 @@ PUT _index_template/my-index-template } } ``` +% TEST[continued] ## Add data to a data stream [add-data-to-data-stream] @@ -350,6 +352,7 @@ POST my-data-stream/_doc "message": "192.0.2.42 - - [06/May/2099:16:21:15 +0000] \"GET /images/bg.jpg HTTP/1.0\" 200 24736" } ``` +% TEST[continued] ## Search and visualize your data [search-visualize-your-data] @@ -408,6 +411,8 @@ GET my-data-stream/_search ] } ``` +% TEST[setup:my_data_stream] +% TEST[teardown:data_stream_cleanup] {{es}} searches are synchronous by default. Searches across frozen data, long time ranges, or large datasets may take longer. Use the [async search API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-async-search-submit) to run searches in the background. For more search options, see [*The search API*](../solutions/search/querying-for-search.md). @@ -459,3 +464,5 @@ POST my-data-stream/_async_search ] } ``` +% TEST[setup:my_data_stream] +% TEST[teardown:data_stream_cleanup] diff --git a/solutions/search/cross-cluster-search.md b/solutions/search/cross-cluster-search.md index db4a49a1b9..901db28188 100644 --- a/solutions/search/cross-cluster-search.md +++ b/solutions/search/cross-cluster-search.md @@ -82,6 +82,8 @@ PUT _cluster/settings } } ``` +% TEST[setup:host] +% TEST[s/35.238.149.\d+:930\d+/${transport_host}/] 1. Since `skip_unavailable` was not set on `cluster_three`, it uses the default of `true`. See the [Optional remote clusters](#skip-unavailable-clusters) section for details. @@ -105,6 +107,8 @@ GET /cluster_one:my-index-000001/_search "_source": ["user.id", "message", "http.response.status_code"] } ``` +% TEST[continued] +% TEST[setup:my_index] The API returns the following response. Note that when you search one or more remote clusters, a `_clusters` section is included to provide information about the search on each cluster. @@ -168,6 +172,13 @@ The API returns the following response. Note that when you search one or more re } } ``` +% TESTRESPONSE[s/"took": 150/"took": "$body.took"/] +% TESTRESPONSE[s/"max_score": 1/"max_score": "$body.hits.max_score"/] +% TESTRESPONSE[s/"_score": 1/"_score": "$body.hits.hits.0._score"/] +% TESTRESPONSE[s/"total": 12/"total": "$body._shards.total"/] +% TESTRESPONSE[s/"successful": 12/"successful": "$body._shards.successful"/] +% TESTRESPONSE[s/"skipped": 0/"skipped": "$body._shards.skipped"/] +% TESTRESPONSE[s/"took": 148/"took": "$body._clusters.details.cluster_one.took"/] 1. This section of counters shows all possible cluster search states and how many cluster searches are currently in that state. The clusters can be one of the following statuses: **running**, **successful** (searches on all shards were successful), **partial** (searches on at least one shard of the cluster was successful and at least one failed), **skipped** (the search failed on a cluster marked with `skip_unavailable`=`true`) or **failed** (the search failed on a cluster marked with `skip_unavailable`=`false`). 2. The `_clusters/details` section shows metadata about the search on each cluster. @@ -196,6 +207,7 @@ GET /my-index-000001,cluster_one:my-index-000001,cluster_two:my-index-000001/_se "_source": ["user.id", "message", "http.response.status_code"] } ``` +% TEST[continued] The API returns the following response: @@ -318,6 +330,21 @@ The API returns the following response: } } ``` +% TESTRESPONSE[s/"took": 150/"took": "$body.took"/] +% TESTRESPONSE[s/"max_score": 1/"max_score": "$body.hits.max_score"/] +% TESTRESPONSE[s/"_score": 1/"_score": "$body.hits.hits.0._score"/] +% TESTRESPONSE[s/"_score": 2/"_score": "$body.hits.hits.1._score"/] +% TESTRESPONSE[s/"total": 28/"total": "$body._shards.total"/] +% TESTRESPONSE[s/"successful": 28/"successful": "$body._shards.successful"/] +% TESTRESPONSE[s/"total": 10/"total": "$body._clusters.details.(local)._shards.total"/] +% TESTRESPONSE[s/"successful": 10/"successful": "$body._clusters.details.(local)._shards.successful"/] +% TESTRESPONSE[s/"took": 21/"took": "$body._clusters.details.(local).took"/] +% TESTRESPONSE[s/"total": 12/"total": "$body._clusters.details.cluster_one._shards.total"/] +% TESTRESPONSE[s/"successful": 12/"successful": "$body._clusters.details.cluster_one._shards.successful"/] +% TESTRESPONSE[s/"took": 48/"took": "$body._clusters.details.cluster_one.took"/] +% TESTRESPONSE[s/"total" : 6/"total": "$body._clusters.details.cluster_two._shards.total"/] +% TESTRESPONSE[s/"successful" : 6/"successful": "$body._clusters.details.cluster_two._shards.successful"/] +% TESTRESPONSE[s/"took": 141/"took": "$body._clusters.details.cluster_two.took"/] 1. The local (querying) cluster is identified as "(local)". 2. This document’s `_index` parameter doesn’t include a cluster name. This means the document came from the local cluster. @@ -343,6 +370,8 @@ POST /my-index-000001,cluster_one:my-index-000001,cluster_two:my-index-000001/_a "_source": ["user.id", "message", "http.response.status_code"] } ``` +% TEST[continued] +% TEST[s/ccs_minimize_roundtrips=true/ccs_minimize_roundtrips=true&wait_for_completion_timeout=100ms&keep_on_completion=true/] The API returns the following response: @@ -399,6 +428,7 @@ The API returns the following response: } } ``` +% TEST[skip: hard to reproduce initial state] 1. The async search id. 2. When `ccs_minimize_roundtrips` = `true` and searches on the remote clusters are still running, this section indicates the number of shards in scope for the local cluster only and any clusters that have finished their search so far. This will be updated to include the total number of shards across all clusters only when the search is completed. When `ccs_minimize_roundtrips`= `false`, the total shard count across all clusters is known up front and will be correct. @@ -414,6 +444,8 @@ If you set `ccs_minimize_roundtrips=true`, then you will also see partial result ```console GET /_async_search/FklQYndoTDJ2VEFlMEVBTzFJMGhJVFEaLVlKYndBWWZSMUdicUc4WVlEaFl4ZzoxNTU= ``` +% TEST[continued s/FklQYndoTDJ2VEFlMEVBTzFJMGhJVFEaLVlKYndBWWZSMUdicUc4WVlEaFl4ZzoxNTU=/${body.id}/] +% TEST[continued s/FklQYndoTDJ2VEFlMEVBTzFJMGhJVFEaLVlKYndBWWZSMUdicUc4WVlEaFl4ZzoxNTU=/${body.id}/] Response: @@ -573,6 +605,25 @@ Response: } } ``` +% TESTRESPONSE[s/FklQYndoTDJ2VEFlMEVBTzFJMGhJVFEaLVlKYndBWWZSMUdicUc4WVlEaFl4ZzoxNTU=/$body.id/] +% TESTRESPONSE[s/"is_partial": true/"is_partial": $body.is_partial/] +% TESTRESPONSE[s/"is_running": true/"is_running": $body.is_running/] +% TESTRESPONSE[s/1685564911108/$body.start_time_in_millis/] +% TESTRESPONSE[s/1685996911108/$body.expiration_time_in_millis/] +% TESTRESPONSE[s/1685564938727/$body.completion_time_in_millis/] +% TESTRESPONSE[s/"took": 27619/"took": "$body.response.took"/] +% TESTRESPONSE[s/"took": 2034/"took": "$body.$_path"/] +% TESTRESPONSE[s/"took": 9039/"took": "$body.$_path"/] +% TESTRESPONSE[s/"took": 27550/"took": "$body.$_path"/] +% TESTRESPONSE[s/"total": 28/"total": $body.response._shards.total/] +% TESTRESPONSE[s/"successful": 28/"successful": $body.response._shards.successful/] +% TESTRESPONSE[s/"successful": 3/"successful": $body.response._clusters.successful/] +% TESTRESPONSE[s/"value": 1067/"value": "$body.response.hits.total.value"/] +% TESTRESPONSE[s/"relation": "eq"/"relation": "$body.response.hits.total.relation"/] +% TESTRESPONSE[s/"max_score": 1.8293576/"max_score": "$body.response.hits.max_score"/] +% TESTRESPONSE[s/"hits": […​list of hits here…​]/"hits": $body.response.hits.hits/] +% TESTRESPONSE[s/"total": \d+/"total": $body.$_path/] +% TESTRESPONSE[s/"successful": \d+/"successful": $body.$_path/] 1. Once the search has finished, the completion_time is present. 2. The `_shards` section is now updated to show that 28 total shards were searched across all clusters and that all were successful. @@ -600,6 +651,7 @@ Here is an example of a search with partial results due to a failure on one shar ```console GET /_async_search/status/FmpwbThueVB4UkRDeUxqb1l4akIza3cbWEJyeVBPQldTV3FGZGdIeUVabXBldzoyMDIw ``` +% TEST[continued s/FmpwbThueVB4UkRDeUxqb1l4akIza3cbWEJyeVBPQldTV3FGZGdIeUVabXBldzoyMDIw/${body.id}/] Response: @@ -706,6 +758,7 @@ Response: } } ``` +% TEST[skip: hard to reproduce failure results] 1. The search results are marked as partial, since at least one shard search failed. 2. The `_shards` section includes shard failure info. @@ -722,6 +775,7 @@ If you want the search to still return results even when a cluster is unavailabl ```console GET /_async_search/FjktRGJ1Y2w1U0phLTRhZnVyeUZ2MVEbWEJyeVBPQldTV3FGZGdIeUVabXBldzo5NzA4 ``` +% TEST[continued s/FjktRGJ1Y2w1U0phLTRhZnVyeUZ2MVEbWEJyeVBPQldTV3FGZGdIeUVabXBldzo5NzA4/${body.id}/] Response: @@ -808,6 +862,7 @@ Response: } } ``` +% TEST[skip: hard to reproduce failure results] 1. The shard accounting will often be only partial when errors like this occur, since we need to be able to get shard info from remote clusters on each search. 2. `cluster_one` disconnected during the search and it returned no results. Since it is marked in the remote cluster configuration as `skip_unavailable`=`true`, its status is "skipped", which will not fail the entire search. @@ -840,6 +895,8 @@ POST /my-index-000001,cluster*:my-index-000001,-cluster_three:*/_async_search < "_source": ["user.id", "message", "http.response.status_code"] } ``` +% TEST[continued] +% TEST[s/ccs_minimize_roundtrips=true/ccs_minimize_roundtrips=true&wait_for_completion_timeout=100ms&keep_on_completion=true/] 1. The `cluster*` notation would naturally include `cluster_one`, `cluster_two` and `cluster_three`. To exclude `cluster_three` use a `-` before the cluster name along with a simple wildcard `*` in the index position. This indicates that you do not want the search to make any contact with `cluster_three`. @@ -859,6 +916,8 @@ POST /my-index-000001,cluster*:my-index-*,cluster_three:-my-index-000001/_async_ "_source": ["user.id", "message", "http.response.status_code"] } ``` +% TEST[continued] +% TEST[s/ccs_minimize_roundtrips=true/ccs_minimize_roundtrips=true&wait_for_completion_timeout=100ms&keep_on_completion=true/] 1. This will **not** exclude `cluster_three` from the search. It will still be contacted and told to search any indexes matching `my-index-*` except for `my-index-000001`. @@ -887,6 +946,8 @@ POST /my-index-000001,cluster_one:my-index-000001,cluster_two:my-index-000001/_a "_source": ["user.id", "message", "http.response.status_code"] } ``` +% TEST[continued] +% TEST[s/ccs_minimize_roundtrips=false/ccs_minimize_roundtrips=false&wait_for_completion_timeout=2s&keep_on_completion=true/] The API returns the following response if the query takes longer than the `wait_for_completion_timeout` duration (see [Async search](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-async-search-submit)). @@ -960,6 +1021,7 @@ The API returns the following response if the query takes longer than the `wait_ } } ``` +% TEST[skip: hard to reproduce intermediate results] 1. All shards from all clusters in scope for the search are listed here. Watch this section and/or the _clusters section for updates to monitor search progress. 2. From the `_clusters` section we can see that all the clusters are in "running" state. diff --git a/solutions/search/full-text/search-relevance/mixing-exact-search-with-stemming.md b/solutions/search/full-text/search-relevance/mixing-exact-search-with-stemming.md index 716c748fcf..89e82f3b16 100644 --- a/solutions/search/full-text/search-relevance/mixing-exact-search-with-stemming.md +++ b/solutions/search/full-text/search-relevance/mixing-exact-search-with-stemming.md @@ -67,6 +67,7 @@ GET index/_search } } ``` +% TEST[continued] ```console-result { @@ -105,6 +106,7 @@ GET index/_search } } ``` +% TESTRESPONSE[s/"took": 2,/"took": "$body.took",/] On the other hand, searching for `ski` on `body.exact` would only return document `1` since the analysis chain of `body.exact` does not perform stemming. @@ -119,6 +121,7 @@ GET index/_search } } ``` +% TEST[continued] ```console-result { @@ -149,6 +152,7 @@ GET index/_search } } ``` +% TESTRESPONSE[s/"took": 1,/"took": "$body.took",/] This is not something that is easy to expose to end users, as we would need to have a way to figure out whether they are looking for an exact match or not and redirect to the appropriate field accordingly. Also what to do if only parts of the query need to be matched exactly while other parts should still take stemming into account? @@ -166,6 +170,7 @@ GET index/_search } } ``` +% TEST[continued] ```console-result { @@ -196,6 +201,7 @@ GET index/_search } } ``` +% TESTRESPONSE[s/"took": 2,/"took": "$body.took",/] In the above case, since `ski` was in-between quotes, it was searched on the `body.exact` field due to the `quote_field_suffix` parameter, so only document `1` matched. This allows users to mix exact search with stemmed search as they like. diff --git a/solutions/search/full-text/search-relevance/static-scoring-signals.md b/solutions/search/full-text/search-relevance/static-scoring-signals.md index 9d4739b816..fc30aef81c 100644 --- a/solutions/search/full-text/search-relevance/static-scoring-signals.md +++ b/solutions/search/full-text/search-relevance/static-scoring-signals.md @@ -34,6 +34,7 @@ GET index/_search } } ``` +% TEST[continued] 1. `pagerank` must be mapped as a [Numeric](elasticsearch://reference/elasticsearch/mapping-reference/number.md) diff --git a/solutions/search/hybrid-semantic-text.md b/solutions/search/hybrid-semantic-text.md index e6db19a531..b3d7cbc180 100644 --- a/solutions/search/hybrid-semantic-text.md +++ b/solutions/search/hybrid-semantic-text.md @@ -81,6 +81,7 @@ POST _reindex?wait_for_completion=false } } ``` +% TEST[skip:TBD] 1. The default batch size for reindexing is 1000. Reducing size to a smaller number makes the update of the reindexing process quicker which enables you to follow the progress closely and detect errors early. @@ -90,6 +91,7 @@ The call returns a task ID to monitor the progress: ```console GET _tasks/ ``` +% TEST[skip:TBD] Reindexing large datasets can take a long time. You can test this workflow using only a subset of the dataset. @@ -98,6 +100,7 @@ To cancel the reindexing process and generate embeddings for the subset that was ```console POST _tasks//_cancel ``` +% TEST[skip:TBD] ## Perform hybrid search [hybrid-search-perform-search] diff --git a/solutions/search/querydsl-full-text-filter-tutorial.md b/solutions/search/querydsl-full-text-filter-tutorial.md index c0625b81dc..569351722d 100644 --- a/solutions/search/querydsl-full-text-filter-tutorial.md +++ b/solutions/search/querydsl-full-text-filter-tutorial.md @@ -31,6 +31,7 @@ You’ll need a running {{es}} cluster, together with {{kib}} to use the Dev Too ```sh curl -fsSL https://elastic.co/start-local | sh ``` +% NOTCONSOLE ## Step 1: Create an index [full-text-filter-tutorial-create-index] @@ -40,6 +41,7 @@ Create the `cooking_blog` index to get started: ```console PUT /cooking_blog ``` +% TESTSETUP Now define the mappings for the index: @@ -99,6 +101,7 @@ PUT /cooking_blog/_mapping } } ``` +% TEST 1. The `standard` analyzer is used by default for `text` fields if an `analyzer` isn’t specified. It’s included here for demonstration purposes. 2. [Multi-fields](elasticsearch://reference/elasticsearch/mapping-reference/multi-fields.md) are used here to index `text` fields as both `text` and `keyword` [data types](elasticsearch://reference/elasticsearch/mapping-reference/field-data-types.md). This enables both full-text search and exact matching/filtering on the same field. Note that if you used [dynamic mapping](../../manage-data/data-store/mapping/dynamic-field-mapping.md), these multi-fields would be created automatically. @@ -129,6 +132,7 @@ POST /cooking_blog/_bulk?refresh=wait_for {"index":{"_id":"5"}} {"title":"Crispy Oven-Fried Chicken","description":"Get that perfect crunch without the deep fryer! This oven-fried chicken recipe delivers crispy, juicy results every time. A healthier take on the classic comfort food.","author":"Maria Rodriguez","date":"2023-05-20","category":"Main Course","tags":["chicken","oven-fried","healthy"],"rating":4.9} ``` +% TEST[continued] ## Step 3: Perform basic full-text searches [full-text-filter-tutorial-match-query] @@ -154,6 +158,7 @@ GET /cooking_blog/_search } } ``` +% TEST[continued] 1. By default, the `match` query uses `OR` logic between the resulting tokens. This means it will match documents that contain either "fluffy" or "pancakes", or both, in the description field. @@ -200,6 +205,12 @@ At search time, {{es}} defaults to the analyzer defined in the field mapping. In } } ``` +% TESTRESPONSE[s/"took": 0/"took": "$body.took"/] +% TESTRESPONSE[s/"total": 1/"total": $body._shards.total/] +% TESTRESPONSE[s/"successful": 1/"successful": $body._shards.successful/] +% TESTRESPONSE[s/"value": 1/"value": $body.hits.total.value/] +% TESTRESPONSE[s/"max_score": 1.8378843/"max_score": $body.hits.max_score/] +% TESTRESPONSE[s/"_score": 1.8378843/"_score": $body.hits.hits.0._score/] 1. The `hits` object contains the total number of matching documents and their relation to the total. 2. `max_score` is the highest relevance score among all matching documents. In this example, we only have one matching document. @@ -229,6 +240,7 @@ GET /cooking_blog/_search } } ``` +% TEST[continued] ::::{dropdown} Example response ```console-result @@ -251,6 +263,7 @@ GET /cooking_blog/_search } } ``` +% TESTRESPONSE[s/"took": 0/"took": "$body.took"/] :::: @@ -275,6 +288,7 @@ GET /cooking_blog/_search } } ``` +% TEST[continued] ## Step 4: Search across multiple fields at once [full-text-filter-tutorial-multi-match] @@ -294,6 +308,7 @@ GET /cooking_blog/_search } } ``` +% TEST[continued] This query searches for "vegetarian curry" across the title, description, and tags fields. Each field is treated with equal importance. @@ -310,6 +325,7 @@ GET /cooking_blog/_search } } ``` +% TEST[continued] 1. The `^` syntax applies a boost to specific fields:* `title^3`: The title field is 3 times more important than an unboosted field * `description^2`: The description is 2 times more important @@ -363,6 +379,9 @@ Learn more about fields and per-field boosting in the [`multi_match` query](elas } } ``` +% TESTRESPONSE[s/"took": 0/"took": "$body.took"/] +% TESTRESPONSE[s/"_score": 7.546015/"_score": $body.hits.hits.0._score/] +% TESTRESPONSE[s/"max_score": 7.546015/"max_score": $body.hits.max_score/] 1. The title contains "Vegetarian" and "Curry", which matches our search terms. The title field has the highest boost (^3), contributing significantly to this document’s relevance score. 2. The description contains "curry" and related terms like "vegetables", further increasing the document’s relevance. @@ -399,6 +418,7 @@ GET /cooking_blog/_search } } ``` +% TEST[continued] 1. Note the use of `category.keyword` here. This refers to the [`keyword`](elasticsearch://reference/elasticsearch/mapping-reference/keyword.md) multi-field of the `category` field, ensuring an exact, case-sensitive match. @@ -430,6 +450,7 @@ GET /cooking_blog/_search } } ``` +% TEST[continued] 1. Greater than or equal to May 1, 2023. 2. Less than or equal to May 31, 2023. @@ -452,6 +473,7 @@ GET /cooking_blog/_search } } ``` +% TEST[continued] 1. The `term` query has zero flexibility. For example, here the queries `maria` or `maria rodriguez` would have zero hits, due to case sensitivity. @@ -525,6 +547,7 @@ GET /cooking_blog/_search } } ``` +% TEST[continued] 1. The `must_not` clause excludes documents that match the specified criteria. This is a powerful tool for filtering out unwanted results. @@ -570,6 +593,7 @@ GET /cooking_blog/_search } } ``` +% TESTRESPONSE[s/"took": 1/"took": "$body.took"/] 1. The title contains "Spicy" and "Curry", matching our should condition. With the default [best_fields](elasticsearch://reference/query-languages/query-dsl-multi-match-query.md#type-best-fields) behavior, this field contributes most to the relevance score. 2. While the description also contains matching terms, only the best matching field’s score is used by default. diff --git a/solutions/search/ranking/learning-to-rank-ltr.md b/solutions/search/ranking/learning-to-rank-ltr.md index b3845dea47..645b55025c 100644 --- a/solutions/search/ranking/learning-to-rank-ltr.md +++ b/solutions/search/ranking/learning-to-rank-ltr.md @@ -80,6 +80,7 @@ To do this in {{es}}, use templated queries to extract features when building th } ] ``` +% NOTCONSOLE ## Models [learning-to-rank-models] diff --git a/solutions/search/ranking/learning-to-rank-model-training.md b/solutions/search/ranking/learning-to-rank-model-training.md index 32e201e63e..329ad3a6ce 100644 --- a/solutions/search/ranking/learning-to-rank-model-training.md +++ b/solutions/search/ranking/learning-to-rank-model-training.md @@ -77,6 +77,7 @@ feature_extractors=[ ), ] ``` +% NOTCONSOLE ::::{admonition} Term statistics as features :class: note @@ -93,6 +94,7 @@ from eland.ml.ltr import LTRModelConfig ltr_config = LTRModelConfig(feature_extractors) ``` +% NOTCONSOLE ### Extracting features for training [learning-to-rank-model-training-feature-extraction] @@ -114,6 +116,7 @@ feature_logger.extract_features( doc_ids=["doc-1", "doc-2"] ) ``` +% NOTCONSOLE Our [example notebook](https://github.com/elastic/elasticsearch-labs/blob/main/notebooks/search/08-learning-to-rank.ipynb) explains how to use the `FeatureLogger` to build a training dataset, by adding features to a judgment list. @@ -141,6 +144,7 @@ MLModel.import_ltr_model( es_if_exists="replace", ) ``` +% NOTCONSOLE This method will serialize the trained model and the Learning To Rank configuration (including feature extraction) in a format that {{es}} can understand. The model is then deployed to {{es}} using the [Create Trained Models API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-ml-put-trained-model). diff --git a/solutions/search/ranking/learning-to-rank-search-usage.md b/solutions/search/ranking/learning-to-rank-search-usage.md index 48509d792c..65cc4fd5ce 100644 --- a/solutions/search/ranking/learning-to-rank-search-usage.md +++ b/solutions/search/ranking/learning-to-rank-search-usage.md @@ -42,6 +42,7 @@ GET my-index/_search } } ``` +% TEST[skip:TBD] 1. First pass query providing documents to be rescored. 2. The unique identifier of the trained model uploaded to {{es}}. diff --git a/solutions/search/ranking/semantic-reranking.md b/solutions/search/ranking/semantic-reranking.md index 3d875b77e1..9970452fb7 100644 --- a/solutions/search/ranking/semantic-reranking.md +++ b/solutions/search/ranking/semantic-reranking.md @@ -128,6 +128,7 @@ POST _search } } ``` +% TEST[skip:TBD] :::: diff --git a/solutions/search/retrievers-overview.md b/solutions/search/retrievers-overview.md index dd50912a89..019ba52e23 100644 --- a/solutions/search/retrievers-overview.md +++ b/solutions/search/retrievers-overview.md @@ -81,6 +81,7 @@ GET example-index/_search } } ``` +% NOTCONSOLE This example demonstrates how you can combine different retrieval strategies into a single `retriever` pipeline. @@ -116,6 +117,7 @@ GET example-index/_search } } ``` +% NOTCONSOLE :::: For more examples, refer to [retriever examples](retrievers-examples.md). diff --git a/solutions/search/run-elasticsearch-locally.md b/solutions/search/run-elasticsearch-locally.md index 8d644576eb..8c46d248cf 100644 --- a/solutions/search/run-elasticsearch-locally.md +++ b/solutions/search/run-elasticsearch-locally.md @@ -31,6 +31,7 @@ To set up {{es}} and {{kib}} locally, run the `start-local` script: ```sh curl -fsSL https://elastic.co/start-local | sh ``` +% NOTCONSOLE This script creates an `elastic-start-local` folder containing configuration files and starts both {{es}} and {{kib}} using Docker. diff --git a/solutions/search/search-applications.md b/solutions/search/search-applications.md index fa0ad18505..3062101c24 100644 --- a/solutions/search/search-applications.md +++ b/solutions/search/search-applications.md @@ -103,6 +103,7 @@ PUT /_application/search_application/my_search_application } } ``` +% TEST[skip:TODO] ### Search templates [search-application-overview-get-started-templates] diff --git a/solutions/search/search-applications/search-application-api.md b/solutions/search/search-applications/search-application-api.md index 4f881ca3f2..935f4bf076 100644 --- a/solutions/search/search-applications/search-application-api.md +++ b/solutions/search/search-applications/search-application-api.md @@ -43,12 +43,15 @@ PUT _application/search_application/my_search_application "indices": ["index1", "index2"] } ``` +% TEST[warning:Using default search application template which is subject to change. We recommend storing a template to avoid breaking changes.] You can then use the [get search application](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search-application-get) API call to view your newly created search application, which will also include the default template that was created for you: ```console GET _application/search_application/my_search_application ``` +% TEST[continued] +% TEST[warning:Using default search application template which is subject to change. We recommend storing a template to avoid breaking changes.] In this case, the response would be: @@ -80,6 +83,7 @@ In this case, the response would be: } } ``` +% TESTRESPONSE[s/"updated_at_millis": 1715802354482/"updated_at_millis": $body.$_path/] The default template is very minimal: @@ -103,6 +107,7 @@ The default template is very minimal: } } ``` +% TEST[skip:This is not a console result but NOTCONSOLE annotation by itself still fails the test] This may be useful for initial exploration of search templates, but you’ll likely want to update this. @@ -118,6 +123,7 @@ You can use the [render search application query](https://www.elastic.co/docs/ap ```console POST _application/search_application/my_search_application/_render_query ``` +% TEST[continued] will return: @@ -132,6 +138,7 @@ will return: } } ``` +% TEST[continued] This uses the default parameters that were defined with the template. You can also specify one or more parameters to the render call, for example: @@ -143,6 +150,7 @@ POST _application/search_application/my_search_application/_render_query } } ``` +% TEST[continued] will return: @@ -157,6 +165,7 @@ will return: } } ``` +% TEST[continued] In this case, the `{{query_string}}` parameter has been replaced with the value `rock climbing`, and the `{{default_field}}` parameter was not specified so used the default value of `*`. @@ -165,6 +174,8 @@ When you actually perform a search with no parameters, it will execute the under ```console POST _application/search_application/my_search_application/_search ``` +% TEST[continued] +% TEST[continued] Searching with the `query_string` and/or `default_field` parameters will perform a [`query_string`](elasticsearch://reference/query-languages/query-dsl-query-string-query.md) query. @@ -196,6 +207,7 @@ POST _application/search_application/my_search_application/_search } } ``` +% TEST[continued] In this example, we’ve overridden the `query_string` parameter’s default value of `*`. Since we didn’t specify `default_field` the value of this parameter will still be `*`. @@ -278,6 +290,7 @@ POST _application/search_application/my_search_application/_search } } ``` +% TEST[continued] The `text_fields` parameters can be overridden with new/different fields and boosts to experiment with the best configuration for your use case. This template also supports pagination and `explain` via parameters. @@ -366,6 +379,7 @@ POST _application/search_application/my-search-app/_search } } ``` +% TEST[skip:ELSER requires inference] ### Text search + ELSER [search-application-api-catchall-template] @@ -476,6 +490,7 @@ POST _application/search_application/my_search_application/_search } } ``` +% TEST[skip:ELSER requires inference] An ELSER search query using this template will look like the following example: @@ -492,6 +507,7 @@ POST _application/search_application/my_search_application/_search } } ``` +% TEST[skip:ELSER requires inference] A combined text search and ELSER search query using this template will look like the following example: @@ -511,6 +527,7 @@ POST _application/search_application/my_search_application/_search } } ``` +% TEST[skip:ELSER requires inference] ::::{tip} Text search results and ELSER search results are expected to have significantly different scores in some cases, which makes ranking challenging. To find the best search result mix for your dataset, we suggest experimenting with the boost values provided in the example template: @@ -597,6 +614,7 @@ POST _application/search_application/my_search_application/_search } } ``` +% TEST[skip:ELSER requires inference] ### kNN search [search-applications-knn-template] @@ -660,6 +678,7 @@ POST _application/search_application/my_search_application/_search } } ``` +% TEST[continued] A template supporting approximate kNN search will look like the following example: @@ -709,4 +728,5 @@ POST _application/search_application/my_search_application/_search } } ``` +% TEST[continued] diff --git a/solutions/search/search-applications/search-application-client.md b/solutions/search/search-applications/search-application-client.md index 337e018564..24b0203ca1 100644 --- a/solutions/search/search-applications/search-application-client.md +++ b/solutions/search/search-applications/search-application-client.md @@ -97,6 +97,7 @@ Use the following import statement: ```js import SearchApplicationClient from '@elastic/search-application-client'; ``` +% NOTCONSOLE Configure the client with your deployment details to start making search requests. You can generate an API key on the **Connect** page in the {{kib}} UI. Go to **Search > Search Applications >** **> Connect**. You’ll find the following information prepopulated to initialize the client: @@ -120,6 +121,7 @@ const results = await request() .query('star wars') .search() ``` +% NOTCONSOLE #### Option 2: Using CDN [search-application-client-client-configuration-import-cdn] @@ -203,6 +205,7 @@ PUT _application/search_application/my-example-app } } ``` +% TEST[skip:TODO] This will allow you to add any template parameters you need to your template and then provide the values in your client request. Use `addParameter` to inject actual values into your template parameters. @@ -214,6 +217,7 @@ const results = await request() .addParameter('search_fields', ['title', 'description']) .search() ``` +% NOTCONSOLE ### Example template [search-application-client-client-template-example] @@ -449,6 +453,7 @@ If a match was found, this will return the results with a highlight field. For e ] } ``` +% NOTCONSOLE #### Highlighting helper [search-application-client-client-features-highlight-helpers] @@ -502,6 +507,7 @@ const results = await request() .setSort([{ year: 'asc' }, '_score']) .search() ``` +% NOTCONSOLE ### Filtering [search-application-client-client-features-filter] @@ -565,6 +571,7 @@ const request = Client( } ) ``` +% NOTCONSOLE ::::{note} In {{es}}, the `keyword` type is used for fields that need to be searchable in their exact, unmodified form. This means these queries are case-sensitive. We use this type for facets because facets require aggregating and filtering data based on exact values or terms. @@ -590,6 +597,7 @@ const results = await request() }) .search() ``` +% NOTCONSOLE You can access the facets in the results: @@ -721,4 +729,5 @@ You can access the facets in the results: ] } ``` +% NOTCONSOLE diff --git a/solutions/search/search-applications/search-application-security.md b/solutions/search/search-applications/search-application-security.md index f557e56a88..d24e15bd1f 100644 --- a/solutions/search/search-applications/search-application-security.md +++ b/solutions/search/search-applications/search-application-security.md @@ -58,6 +58,7 @@ POST /_security/api_key } } ``` +% TEST[skip:TODO] 1. `indices.name` must be the name(s) of the Search Application(s), not the underlying {{es}} indices. 2. `restriction.workflows` must be set to the concrete value `search_application_query`. @@ -80,6 +81,7 @@ The response will look like this: "encoded": "djFDQ0pZa0J2YjVQZzlULV9KZ086enRWSS0xUTRSalM4cUZEeEFWZXQ1dw" } ``` +% TEST[skip:TODO] The encoded value can then be directly used in the Authorization header. Here’s an example using cURL: @@ -95,6 +97,7 @@ curl -XPOST "http://localhost:9200/_application/search_application/website-produ } }' ``` +% NOTCONSOLE ::::{tip} If `expiration` is not present, by default {{es}} API keys never expire. The API key can be invalidated using the [invalidate API key API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-security-invalidate-api-key). @@ -167,6 +170,7 @@ PUT _application/search_application/website-product-search } } ``` +% TEST[skip:TODO] Using that definition, the Search Application Search API performs the following parameter validation: diff --git a/solutions/search/search-templates.md b/solutions/search/search-templates.md index 8755a4c6bf..1beb82fd84 100644 --- a/solutions/search/search-templates.md +++ b/solutions/search/search-templates.md @@ -62,6 +62,8 @@ POST _render/template } } ``` +% TEST[continued] +% TEST[continued] When rendered, the template outputs a [search request body](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search). @@ -100,6 +102,7 @@ POST _render/template } } ``` +% TEST[continued] ### Run a templated search [run-templated-search] @@ -117,6 +120,7 @@ GET my-index/_search/template } } ``` +% TEST[continued] The response uses the same properties as the [search API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search)'s response. @@ -149,6 +153,7 @@ The response uses the same properties as the [search API](https://www.elastic.co } } ``` +% TESTRESPONSE[s/"took": 36/"took": "$body.took"/] ### Run multiple templated searches [run-multiple-templated-searches] @@ -162,6 +167,8 @@ GET my-index/_msearch/template { } { "id": "my-other-search-template", "params": { "query_type": "match_all" }} ``` +% TEST[continued] +% TEST[s/my-other-search-template/my-search-template/] ### Get search templates [get-search-templates] @@ -171,12 +178,14 @@ To retrieve a search template, use the [get stored script API](https://www.elast ```console GET _scripts/my-search-template ``` +% TEST[continued] To get a list of all search templates and other stored scripts, use the [cluster state API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-cluster-state). ```console GET _cluster/state/metadata?pretty&filter_path=metadata.stored_scripts ``` +% TEST[continued] ### Delete a search template [delete-search-template] @@ -186,6 +195,7 @@ To delete a search template, use the [delete stored script API](https://www.elas ```console DELETE _scripts/my-search-template ``` +% TEST[continued] ### Set default values [search-template-set-default-values] @@ -617,6 +627,7 @@ POST _render/template } } ``` +% TEST[continued] When rendered, template outputs as: @@ -726,6 +737,7 @@ POST _render/template } } ``` +% TEST[continued] When rendered, template outputs: @@ -787,6 +799,7 @@ POST _render/template } } ``` +% TEST[continued] When rendered the template outputs: @@ -913,6 +926,7 @@ POST _render/template } } ``` +% TEST[continued] When rendered, template outputs: diff --git a/solutions/search/semantic-search/cohere-es.md b/solutions/search/semantic-search/cohere-es.md index 7fe25a4cc4..d73dc9ba35 100644 --- a/solutions/search/semantic-search/cohere-es.md +++ b/solutions/search/semantic-search/cohere-es.md @@ -318,3 +318,4 @@ Response: Biosimilarity is based on the comparability concept, which has been us Sources: Interchangeability of Biosimilars: A European Perspective: (...) ``` +% NOTCONSOLE diff --git a/solutions/search/semantic-search/semantic-search-elser-ingest-pipelines.md b/solutions/search/semantic-search/semantic-search-elser-ingest-pipelines.md index 7a9ac7286f..a7a29ea1a5 100644 --- a/solutions/search/semantic-search/semantic-search-elser-ingest-pipelines.md +++ b/solutions/search/semantic-search/semantic-search-elser-ingest-pipelines.md @@ -59,6 +59,7 @@ PUT my-index } } ``` +% TEST[skip:TBD] 1. The name of the field to contain the generated tokens. It must be referenced in the {{infer}} pipeline configuration in the next step. 2. The field to contain the tokens is a `sparse_vector` field. @@ -122,6 +123,7 @@ POST _reindex?wait_for_completion=false } } ``` +% TEST[skip:TBD] 1. The default batch size for reindexing is 1000. Reducing `size` to a smaller number makes the update of the reindexing process quicker which enables you to follow the progress closely and detect errors early. @@ -131,6 +133,7 @@ The call returns a task ID to monitor the progress: ```console GET _tasks/ ``` +% TEST[skip:TBD] You can also open the Trained Models UI, select the Pipelines tab under ELSER to follow the progress. @@ -139,6 +142,7 @@ Reindexing large datasets can take a long time. You can test this workflow using ```console POST _tasks//_cancel ``` +% TEST[skip:TBD] ### Semantic search by using the `sparse_vector` query [text-expansion-query] @@ -157,6 +161,7 @@ GET my-index/_search } } ``` +% TEST[skip:TBD] The result is the top 10 documents that are closest in meaning to your query text from the `my-index` index sorted by their relevancy. The result also contains the extracted tokens for each of the relevant search results with their weights. Tokens are learned associations capturing relevance, they are not synonyms. To learn more about what tokens are, refer to [this page](/explore-analyze/machine-learning/nlp/ml-nlp-elser.md#elser-tokens). It is possible to exclude tokens from source, refer to [this section](#save-space) to learn more. @@ -197,6 +202,7 @@ The result is the top 10 documents that are closest in meaning to your query tex ] } ``` +% NOTCONSOLE ### Combining semantic search with other queries [text-expansion-compound-query] @@ -230,6 +236,7 @@ GET my-index/_search "min_score": 10 <4> } ``` +% TEST[skip:TBD] 1. Both the `sparse_vector` and the `query_string` queries are in a `should` clause of a `bool` query. 2. The `boost` value is `1` for the `sparse_vector` query which is the default value. This means that the relevance score of the results of this query are not boosted. @@ -272,6 +279,7 @@ PUT my-index } } ``` +% TEST[skip:TBD] ::::{note} Depending on your data, the `sparse_vector` query may be faster with `track_total_hits: false`. diff --git a/solutions/search/semantic-search/semantic-search-inference.md b/solutions/search/semantic-search/semantic-search-inference.md index b3561385d9..fb37be86a3 100644 --- a/solutions/search/semantic-search/semantic-search-inference.md +++ b/solutions/search/semantic-search/semantic-search-inference.md @@ -108,6 +108,7 @@ PUT _inference/text_embedding/cohere_embeddings <1> } } ``` +% TEST[skip:TBD] 1. The task type is `text_embedding` in the path and the `inference_id` which is the unique identifier of the {{infer}} endpoint is `cohere_embeddings`. 2. The API key of your Cohere account. You can find your API keys in your Cohere dashboard under the [API keys section](https://dashboard.cohere.com/api-keys). You need to provide your API key only once. The [Get {{infer}} API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-get) does not return your API key. @@ -130,6 +131,7 @@ PUT _inference/sparse_embedding/elser_embeddings <1> } } ``` +% TEST[skip:TBD] 1. The task type is `sparse_embedding` in the path and the `inference_id` which is the unique identifier of the {{infer}} endpoint is `elser_embeddings`. @@ -155,6 +157,7 @@ PUT _inference/text_embedding/hugging_face_embeddings <1> } } ``` +% TEST[skip:TBD] 1. The task type is `text_embedding` in the path and the `inference_id` which is the unique identifier of the {{infer}} endpoint is `hugging_face_embeddings`. 2. A valid HuggingFace access token. You can find on the [settings page of your account](https://huggingface.co/settings/tokens). @@ -172,6 +175,7 @@ PUT _inference/text_embedding/openai_embeddings <1> } } ``` +% TEST[skip:TBD] 1. The task type is `text_embedding` in the path and the `inference_id` which is the unique identifier of the {{infer}} endpoint is `openai_embeddings`. 2. The API key of your OpenAI account. You can find your OpenAI API keys in your OpenAI account under the [API keys section](https://platform.openai.com/api-keys). You need to provide your API key only once. The [Get {{infer}} API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-get) does not return your API key. @@ -196,6 +200,7 @@ PUT _inference/text_embedding/azure_openai_embeddings <1> } } ``` +% TEST[skip:TBD] 1. The task type is `text_embedding` in the path and the `inference_id` which is the unique identifier of the {{infer}} endpoint is `azure_openai_embeddings`. 2. The API key for accessing your Azure OpenAI services. Alternately, you can provide an `entra_id` instead of an `api_key` here. The [Get {{infer}} API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-get) does not return this information. @@ -221,6 +226,7 @@ PUT _inference/text_embedding/azure_ai_studio_embeddings <1> } } ``` +% TEST[skip:TBD] 1. The task type is `text_embedding` in the path and the `inference_id` which is the unique identifier of the {{infer}} endpoint is `azure_ai_studio_embeddings`. 2. The API key for accessing your Azure AI Studio deployed model. You can find this on your model deployment’s overview page. @@ -247,6 +253,7 @@ PUT _inference/text_embedding/google_vertex_ai_embeddings <1> } } ``` +% TEST[skip:TBD] 1. The task type is `text_embedding` per the path. `google_vertex_ai_embeddings` is the unique identifier of the {{infer}} endpoint (its `inference_id`). 2. A valid service account in JSON format for the Google Vertex AI API. @@ -266,6 +273,7 @@ PUT _inference/text_embedding/mistral_embeddings <1> } } ``` +% TEST[skip:TBD] 1. The task type is `text_embedding` in the path and the `inference_id` which is the unique identifier of the {{infer}} endpoint is `mistral_embeddings`. 2. The API key for accessing the Mistral API. You can find this in your Mistral account’s API Keys page. @@ -286,6 +294,7 @@ PUT _inference/text_embedding/amazon_bedrock_embeddings <1> } } ``` +% TEST[skip:TBD] 1. The task type is `text_embedding` in the path and the `inference_id` which is the unique identifier of the {{infer}} endpoint is `amazon_bedrock_embeddings`. 2. The access key can be found on your AWS IAM management page for the user account to access Amazon Bedrock. @@ -308,6 +317,7 @@ PUT _inference/text_embedding/alibabacloud_ai_search_embeddings <1> } } ``` +% TEST[skip:TBD] 1. The task type is `text_embedding` in the path and the `inference_id` which is the unique identifier of the {{infer}} endpoint is `alibabacloud_ai_search_embeddings`. 2. The API key for accessing the AlibabaCloud AI Search API. You can find your API keys in your AlibabaCloud account under the [API keys section](https://opensearch.console.aliyun.com/cn-shanghai/rag/api-key). You need to provide your API key only once. The [Get {{infer}} API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-get) does not return your API key. @@ -852,6 +862,7 @@ POST _reindex?wait_for_completion=false } } ``` +% TEST[skip:TBD] 1. The default batch size for reindexing is 1000. Reducing `size` to a smaller number makes the update of the reindexing process quicker which enables you to follow the progress closely and detect errors early. @@ -875,6 +886,7 @@ POST _reindex?wait_for_completion=false } } ``` +% TEST[skip:TBD] 1. The default batch size for reindexing is 1000. Reducing `size` to a smaller number makes the update of the reindexing process quicker which enables you to follow the progress closely and detect errors early. :::::: @@ -893,6 +905,7 @@ POST _reindex?wait_for_completion=false } } ``` +% TEST[skip:TBD] 1. The default batch size for reindexing is 1000. Reducing `size` to a smaller number makes the update of the reindexing process quicker which enables you to follow the progress closely and detect errors early. :::::: @@ -911,6 +924,7 @@ POST _reindex?wait_for_completion=false } } ``` +% TEST[skip:TBD] 1. The default batch size for reindexing is 1000. Reducing `size` to a smaller number makes the update of the reindexing process quicker which enables you to follow the progress closely and detect errors early. @@ -934,6 +948,7 @@ POST _reindex?wait_for_completion=false } } ``` +% TEST[skip:TBD] 1. The default batch size for reindexing is 1000. Reducing `size` to a smaller number makes the update of the reindexing process quicker which enables you to follow the progress closely and detect errors early. @@ -957,6 +972,7 @@ POST _reindex?wait_for_completion=false } } ``` +% TEST[skip:TBD] 1. The default batch size for reindexing is 1000. Reducing `size` to a smaller number makes the update of the reindexing process quicker which enables you to follow the progress closely and detect errors early. @@ -980,6 +996,7 @@ POST _reindex?wait_for_completion=false } } ``` +% TEST[skip:TBD] 1. The default batch size for reindexing is 1000. Reducing `size` will make updates to the reindexing process faster. This enables you to follow the progress closely and detect errors early. :::::: @@ -998,6 +1015,7 @@ POST _reindex?wait_for_completion=false } } ``` +% TEST[skip:TBD] 1. The default batch size for reindexing is 1000. Reducing `size` to a smaller number makes the update of the reindexing process quicker which enables you to follow the progress closely and detect errors early. :::::: @@ -1016,6 +1034,7 @@ POST _reindex?wait_for_completion=false } } ``` +% TEST[skip:TBD] 1. The default batch size for reindexing is 1000. Reducing `size` to a smaller number makes the update of the reindexing process quicker which enables you to follow the progress closely and detect errors early. :::::: @@ -1034,6 +1053,7 @@ POST _reindex?wait_for_completion=false } } ``` +% TEST[skip:TBD] 1. The default batch size for reindexing is 1000. Reducing `size` to a smaller number makes the update of the reindexing process quicker which enables you to follow the progress closely and detect errors early. :::::: @@ -1044,12 +1064,14 @@ The call returns a task ID to monitor the progress: ```console GET _tasks/ ``` +% TEST[skip:TBD] Reindexing large datasets can take a long time. You can test this workflow using only a subset of the dataset. Do this by cancelling the reindexing process, and only generating embeddings for the subset that was reindexed. The following API request will cancel the reindexing task: ```console POST _tasks//_cancel ``` +% TEST[skip:TBD] ## Semantic search [infer-semantic-search] @@ -1084,6 +1106,7 @@ GET cohere-embeddings/_search ] } ``` +% TEST[skip:TBD] As a result, you receive the top 10 documents that are closest in meaning to the query from the `cohere-embeddings` index sorted by their proximity to the query: @@ -1128,6 +1151,8 @@ As a result, you receive the top 10 documents that are closest in meaning to the (...) ] ``` +% NOTCONSOLE +:::::: :::::: ::::::{tab-item} ELSER @@ -1147,6 +1172,7 @@ GET elser-embeddings/_search ] } ``` +% TEST[skip:TBD] As a result, you receive the top 10 documents that are closest in meaning to the query from the `cohere-embeddings` index sorted by their proximity to the query: @@ -1182,6 +1208,8 @@ As a result, you receive the top 10 documents that are closest in meaning to the (...) ] ``` +% NOTCONSOLE +:::::: :::::: ::::::{tab-item} HuggingFace @@ -1205,6 +1233,7 @@ GET hugging-face-embeddings/_search ] } ``` +% TEST[skip:TBD] As a result, you receive the top 10 documents that are closest in meaning to the query from the `hugging-face-embeddings` index sorted by their proximity to the query: @@ -1249,6 +1278,8 @@ As a result, you receive the top 10 documents that are closest in meaning to the (...) ] ``` +% NOTCONSOLE +:::::: :::::: ::::::{tab-item} OpenAI @@ -1272,6 +1303,7 @@ GET openai-embeddings/_search ] } ``` +% TEST[skip:TBD] As a result, you receive the top 10 documents that are closest in meaning to the query from the `openai-embeddings` index sorted by their proximity to the query: @@ -1307,6 +1339,8 @@ As a result, you receive the top 10 documents that are closest in meaning to the (...) ] ``` +% NOTCONSOLE +:::::: :::::: ::::::{tab-item} Azure OpenAI @@ -1330,6 +1364,7 @@ GET azure-openai-embeddings/_search ] } ``` +% TEST[skip:TBD] As a result, you receive the top 10 documents that are closest in meaning to the query from the `azure-openai-embeddings` index sorted by their proximity to the query: @@ -1365,6 +1400,8 @@ As a result, you receive the top 10 documents that are closest in meaning to the (...) ] ``` +% NOTCONSOLE +:::::: :::::: ::::::{tab-item} Azure AI Studio @@ -1388,6 +1425,7 @@ GET azure-ai-studio-embeddings/_search ] } ``` +% TEST[skip:TBD] As a result, you receive the top 10 documents that are closest in meaning to the query from the `azure-ai-studio-embeddings` index sorted by their proximity to the query: @@ -1423,6 +1461,8 @@ As a result, you receive the top 10 documents that are closest in meaning to the (...) ] ``` +% NOTCONSOLE +:::::: :::::: ::::::{tab-item} Google Vertex AI @@ -1446,6 +1486,7 @@ GET google-vertex-ai-embeddings/_search ] } ``` +% TEST[skip:TBD] As a result, you receive the top 10 documents that are closest in meaning to the query from the `mistral-embeddings` index sorted by their proximity to the query: @@ -1481,6 +1522,8 @@ As a result, you receive the top 10 documents that are closest in meaning to the (...) ] ``` +% NOTCONSOLE +:::::: :::::: ::::::{tab-item} Mistral @@ -1504,6 +1547,7 @@ GET mistral-embeddings/_search ] } ``` +% TEST[skip:TBD] As a result, you receive the top 10 documents that are closest in meaning to the query from the `mistral-embeddings` index sorted by their proximity to the query: @@ -1539,6 +1583,8 @@ As a result, you receive the top 10 documents that are closest in meaning to the (...) ] ``` +% NOTCONSOLE +:::::: :::::: ::::::{tab-item} Amazon Bedrock @@ -1562,6 +1608,7 @@ GET amazon-bedrock-embeddings/_search ] } ``` +% TEST[skip:TBD] As a result, you receive the top 10 documents that are closest in meaning to the query from the `amazon-bedrock-embeddings` index sorted by their proximity to the query: @@ -1597,6 +1644,8 @@ As a result, you receive the top 10 documents that are closest in meaning to the (...) ] ``` +% NOTCONSOLE +:::::: :::::: ::::::{tab-item} AlibabaCloud AI Search @@ -1620,6 +1669,7 @@ GET alibabacloud-ai-search-embeddings/_search ] } ``` +% TEST[skip:TBD] As a result, you receive the top 10 documents that are closest in meaning to the query from the `alibabacloud-ai-search-embeddings` index sorted by their proximity to the query: @@ -1655,6 +1705,8 @@ As a result, you receive the top 10 documents that are closest in meaning to the (...) ] ``` +% NOTCONSOLE +:::::: :::::: ::::::: diff --git a/solutions/search/semantic-search/semantic-search-semantic-text.md b/solutions/search/semantic-search/semantic-search-semantic-text.md index f00f157ad3..f265603e95 100644 --- a/solutions/search/semantic-search/semantic-search-semantic-text.md +++ b/solutions/search/semantic-search/semantic-search-semantic-text.md @@ -45,6 +45,7 @@ PUT semantic-embeddings } } ``` +% TEST[skip:TBD] 1. The name of the field to contain the generated embeddings. 2. The field to contain the embeddings is a `semantic_text` field. Since no `inference_id` is provided, the default endpoint `.elser-2-elasticsearch` for the [`elasticsearch` service](../../../explore-analyze/elastic-inference/inference-api/elasticsearch-inference-integration.md) is used. To use a different {{infer}} service, you must create an {{infer}} endpoint first using the [Create {{infer}} API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put) and then specify it in the `semantic_text` field mapping using the `inference_id` parameter. @@ -88,6 +89,7 @@ POST _reindex?wait_for_completion=false } } ``` +% TEST[skip:TBD] 1. The default batch size for reindexing is 1000. Reducing size to a smaller number makes the update of the reindexing process quicker which enables you to follow the progress closely and detect errors early. @@ -97,12 +99,14 @@ The call returns a task ID to monitor the progress: ```console GET _tasks/ ``` +% TEST[skip:TBD] Reindexing large datasets can take a long time. You can test this workflow using only a subset of the dataset. Do this by cancelling the reindexing process, and only generating embeddings for the subset that was reindexed. The following API request will cancel the reindexing task: ```console POST _tasks//_cancel ``` +% TEST[skip:TBD] ## Semantic search [semantic-text-semantic-search] @@ -120,6 +124,7 @@ GET semantic-embeddings/_search } } ``` +% TEST[skip:TBD] 1. The `semantic_text` field on which you want to perform the search. 2. The query text. diff --git a/solutions/search/vector/bring-own-vectors.md b/solutions/search/vector/bring-own-vectors.md index 21b347241c..aa50656081 100644 --- a/solutions/search/vector/bring-own-vectors.md +++ b/solutions/search/vector/bring-own-vectors.md @@ -54,6 +54,7 @@ PUT /amazon-reviews } } ``` +% TEST SETUP 1. The `dims` parameter must match the length of the embedding vector. Here we’re using a simple 8-dimensional embedding for readability. If not specified, `dims` will be dynamically calculated based on the first indexed document. 2. The `index` parameter is set to `true` to enable the use of the `knn` query. @@ -75,6 +76,7 @@ PUT /amazon-reviews/_doc/1 "review_vector": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8] <1> } ``` +% TEST 1. The size of the `review_vector` array is 8, matching the `dims` count specified in the mapping. @@ -97,6 +99,7 @@ POST /_bulk { "index": { "_index": "amazon-reviews", "_id": "5" } } { "review_text": "This product has ruined my life and the lives of my family and friends.", "review_vector": [0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1] } ``` +% TEST[continued] ## Step 3: Search documents with embeddings [bring-your-own-vectors-search-documents] @@ -116,6 +119,7 @@ POST /amazon-reviews/_search } } ``` +% TEST[skip:flakeyknnerror] 1. In this simple example, we’re sending a raw vector as the query text. In a real-world scenario, you’ll need to generate vectors for queries using an embedding model. 2. The `k` parameter specifies the number of results to return. diff --git a/solutions/search/vector/dense-versus-sparse-ingest-pipelines.md b/solutions/search/vector/dense-versus-sparse-ingest-pipelines.md index d4d2e2ffd8..0e3dc537bf 100644 --- a/solutions/search/vector/dense-versus-sparse-ingest-pipelines.md +++ b/solutions/search/vector/dense-versus-sparse-ingest-pipelines.md @@ -199,6 +199,8 @@ GET my-index/_search } } ``` +% TEST[skip:TBD] +:::::: :::::: ::::::{tab-item} Dense vector models @@ -220,6 +222,8 @@ GET my-index/_search } } ``` +% TEST[skip:TBD] +:::::: :::::: ::::::: @@ -266,6 +270,8 @@ GET my-index/_search } } ``` +% TEST[skip:TBD] +:::::: :::::: ::::::{tab-item} Dense vector models @@ -308,6 +314,8 @@ GET my-index/_search } } ``` +% TEST[skip:TBD] +:::::: :::::: ::::::: diff --git a/troubleshoot/elasticsearch/add-tier.md b/troubleshoot/elasticsearch/add-tier.md index faa2e0c86e..851cf6a7f5 100644 --- a/troubleshoot/elasticsearch/add-tier.md +++ b/troubleshoot/elasticsearch/add-tier.md @@ -69,6 +69,7 @@ To determine which tier an index requires for assignment, use the [get index set ```console GET /my-index-000001/_settings/index.routing.allocation.include._tier_preference?flat_settings ``` +% TEST[continued] The response will look like this: @@ -81,6 +82,7 @@ The response will look like this: } } ``` +% TESTRESPONSE[skip:the result is for illustrating purposes only] 1. Represents a comma-separated list of data tier node roles this index is allowed to be allocated on, the first one in the list being the one with the higher priority i.e. the tier the index is targeting. e.g. in this example the tier preference is `data_warm,data_hot` so the index is targeting the `warm` tier and more nodes with the `data_warm` role are needed in the {{es}} cluster. :::::: diff --git a/troubleshoot/elasticsearch/circuit-breaker-errors.md b/troubleshoot/elasticsearch/circuit-breaker-errors.md index 6d65abd6b8..2aaf567021 100644 --- a/troubleshoot/elasticsearch/circuit-breaker-errors.md +++ b/troubleshoot/elasticsearch/circuit-breaker-errors.md @@ -36,6 +36,7 @@ If a request triggers a circuit breaker, {{es}} returns an error with a `429` HT 'status': 429 } ``` +% NOTCONSOLE {{es}} also writes circuit breaker errors to [`elasticsearch.log`](../../deploy-manage/monitor/logging-configuration/elasticsearch-log4j-configuration-self-managed.md). This is helpful when automated processes, such as allocation, trigger a circuit breaker. @@ -77,3 +78,4 @@ If you’ve triggered the fielddata circuit breaker and can’t disable fielddat ```console POST _cache/clear?fielddata=true ``` +% TEST[s/^/PUT my-index\n/] diff --git a/troubleshoot/elasticsearch/fix-watermark-errors.md b/troubleshoot/elasticsearch/fix-watermark-errors.md index 9a860fc85e..9ebb5ae769 100644 --- a/troubleshoot/elasticsearch/fix-watermark-errors.md +++ b/troubleshoot/elasticsearch/fix-watermark-errors.md @@ -41,6 +41,8 @@ GET _cluster/allocation/explain "primary": false } ``` +% TEST[s/^/PUT my-index\n/] +% TEST[s/"primary": false,/"primary": false/] ## Temporary Relief [fix-watermark-errors-temporary] @@ -67,6 +69,7 @@ PUT */_settings?expand_wildcards=all "index.blocks.read_only_allow_delete": null } ``` +% TEST[s/^/PUT my-index\n/] When a long-term solution is in place, to reset or reconfigure the disk watermarks: diff --git a/troubleshoot/elasticsearch/high-cpu-usage.md b/troubleshoot/elasticsearch/high-cpu-usage.md index c04fa2c739..65c7d0b42b 100644 --- a/troubleshoot/elasticsearch/high-cpu-usage.md +++ b/troubleshoot/elasticsearch/high-cpu-usage.md @@ -63,6 +63,7 @@ If a node has high CPU usage, use the [nodes hot threads API](https://www.elasti ```console GET _nodes/hot_threads ``` +% TEST[s//my-node,my-other-node//] This API returns a breakdown of any hot threads in plain text. High CPU usage frequently correlates to [a long-running task, or a backlog of tasks](task-queue-backlog.md). @@ -113,6 +114,7 @@ The response’s `description` contains the search request and its queries. `run } } ``` +% TESTRESPONSE[skip: no way to get tasks] To cancel a search and free up resources, use the API’s `_cancel` endpoint. diff --git a/troubleshoot/elasticsearch/high-jvm-memory-pressure.md b/troubleshoot/elasticsearch/high-jvm-memory-pressure.md index a1a79c79f1..db8817f0f5 100644 --- a/troubleshoot/elasticsearch/high-jvm-memory-pressure.md +++ b/troubleshoot/elasticsearch/high-jvm-memory-pressure.md @@ -94,6 +94,7 @@ PUT _cluster/settings } } ``` +% TEST[s/^/PUT my-index\n/] **Prevent mapping explosions** diff --git a/troubleshoot/elasticsearch/hotspotting.md b/troubleshoot/elasticsearch/hotspotting.md index 50789a00fe..2c9fd70685 100644 --- a/troubleshoot/elasticsearch/hotspotting.md +++ b/troubleshoot/elasticsearch/hotspotting.md @@ -38,6 +38,7 @@ node_1 * hirstm 24 20 95 node_2 - hirstm 23 18 18 node_3 - hirstmv 25 90 10 ``` +% TEST[skip:illustrative response only] Here we see two significantly unique utilizations: where the master node is at `cpu: 95` and a hot node is at `disk.used_percent: 90%`. This would indicate hot spotting was occurring on these two nodes, and not necessarily from the same root cause. @@ -81,6 +82,7 @@ node_1 446 19 154.8gb 173.1gb node_2 31 52 44.6gb 372.7gb node_3 445 43 271.5gb 289.4gb ``` +% TEST[skip:illustrative response only] Here we see two significantly unique situations. `node_2` has recently restarted, so it has a much lower number of shards than all other nodes. This also relates to `disk.indices` being much smaller than `disk.used` while shards are recovering as seen via [cat recovery](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-cat-recovery). While `node_2`'s shard count is low, it may become a write hot spot due to ongoing [ILM rollovers](elasticsearch://reference/elasticsearch/index-lifecycle-actions/ilm-rollover.md). This is a common root cause of write hot spots covered in the next section. @@ -114,6 +116,7 @@ write node_1 100 3 0 4259 write node_2 0 4 0 980 write node_3 1 5 0 8714 ``` +% TEST[skip:illustrative response only] Here you can see two significantly unique situations. Firstly, `node_1` has a severely backed up write queue compared to other nodes. Secondly, `node_3` shows historically completed writes that are double any other node. These are both probably due to either poorly distributed write-heavy indices, or to multiple write-heavy indices allocated to the same node. Since primary and replica writes are majorly the same amount of cluster work, we usually recommend setting [`index.routing.allocation.total_shards_per_node`](elasticsearch://reference/elasticsearch/index-settings/total-shards-per-node.md#total-shards-per-node) to force index spreading after lining up index shard counts to total nodes. @@ -162,6 +165,7 @@ type action running_time node cancellable direct indices:data/read/eql 10m node_1 true ... ``` +% TEST[skip:illustrative response only] This surfaces a problematic [EQL query](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-eql-search). We can gain further insight on it via [the task management API](https://www.elastic.co/docs/api/doc/elasticsearch/group/endpoint-tasks), diff --git a/troubleshoot/elasticsearch/increase-cluster-shard-limit.md b/troubleshoot/elasticsearch/increase-cluster-shard-limit.md index 027e956427..c87a43587c 100644 --- a/troubleshoot/elasticsearch/increase-cluster-shard-limit.md +++ b/troubleshoot/elasticsearch/increase-cluster-shard-limit.md @@ -76,6 +76,7 @@ To inspect which tier is an index targeting for assignment, use the [get index s ```console GET /my-index-000001/_settings/index.routing.allocation.include._tier_preference?flat_settings ``` +% TEST[continued] The response will look like this: @@ -88,6 +89,7 @@ The response will look like this: } } ``` +% TESTRESPONSE[skip:the result is for illustrating purposes only] 1. Represents a comma separated list of data tier node roles this index is allowed to be allocated on, the first one in the list being the one with the higher priority i.e. the tier the index is targeting. e.g. in this example the tier preference is `data_warm,data_hot` so the index is targeting the `warm` tier and more nodes with the `data_warm` role are needed in the {{es}} cluster. diff --git a/troubleshoot/elasticsearch/increase-shard-limit.md b/troubleshoot/elasticsearch/increase-shard-limit.md index 271d67a377..489efd7d55 100644 --- a/troubleshoot/elasticsearch/increase-shard-limit.md +++ b/troubleshoot/elasticsearch/increase-shard-limit.md @@ -77,6 +77,7 @@ To inspect which tier is an index targeting for assignment, use the [get index s ```console GET /my-index-000001/_settings/index.routing.allocation.include._tier_preference?flat_settings ``` +% TEST[continued] The response will look like this: @@ -89,6 +90,7 @@ The response will look like this: } } ``` +% TESTRESPONSE[skip:the result is for illustrating purposes only] 1. Represents a comma separated list of data tier node roles this index is allowed to be allocated on, the first one in the list being the one with the higher priority i.e. the tier the index is targeting. e.g. in this example the tier preference is `data_warm,data_hot` so the index is targeting the `warm` tier and more nodes with the `data_warm` role are needed in the {{es}} cluster. diff --git a/troubleshoot/elasticsearch/increase-tier-capacity.md b/troubleshoot/elasticsearch/increase-tier-capacity.md index 05369882e3..16a0368063 100644 --- a/troubleshoot/elasticsearch/increase-tier-capacity.md +++ b/troubleshoot/elasticsearch/increase-tier-capacity.md @@ -39,6 +39,8 @@ To inspect which tier an index is targeting for assignment, use the [get index s ```console GET /my-index-000001/_settings/index.routing.allocation.include._tier_preference?flat_settings ``` +% TEST[continued] +% TESTRESPONSE[skip:the result is for illustrating purposes only as don’t want to change a cluster-wide setting] The response will look like this: @@ -51,6 +53,8 @@ The response will look like this: } } ``` +% TESTRESPONSE[skip:the result is for illustrating purposes only] +% TESTRESPONSE[skip:the result is for illustrating purposes only] 1. Represents a comma separated list of data tier node roles this index is allowed to be allocated on, the first one in the list being the one with the higher priority i.e. the tier the index is targeting. e.g. in this example the tier preference is `data_warm,data_hot` so the index is targeting the `warm` tier and more nodes with the `data_warm` role are needed in the {{es}} cluster. diff --git a/troubleshoot/elasticsearch/index-lifecycle-management-errors.md b/troubleshoot/elasticsearch/index-lifecycle-management-errors.md index 0e71e6a062..746e073ccc 100644 --- a/troubleshoot/elasticsearch/index-lifecycle-management-errors.md +++ b/troubleshoot/elasticsearch/index-lifecycle-management-errors.md @@ -32,6 +32,7 @@ PUT _ilm/policy/shrink-index } } ``` +% TEST There is nothing that prevents you from applying the `shrink-index` policy to a new index that has only two shards: @@ -44,6 +45,7 @@ PUT /my-index-000001 } } ``` +% TEST[continued] After five days, {{ilm-init}} attempts to shrink `my-index-000001` from two shards to four shards. Because the shrink action cannot *increase* the number of shards, this operation fails and {{ilm-init}} moves `my-index-000001` to the `ERROR` step. @@ -52,6 +54,7 @@ You can use the [{{ilm-init}} Explain API](https://www.elastic.co/docs/api/doc/e ```console GET /my-index-000001/_ilm/explain ``` +% TEST[continued] Which returns the following information: @@ -94,6 +97,7 @@ Which returns the following information: } } ``` +% TESTRESPONSE[skip:no way to know if we will get this response immediately] 1. The policy being used to manage the index: `shrink-index` 2. The index age: 5.1 days @@ -124,6 +128,7 @@ PUT _ilm/policy/shrink-index } } ``` +% TEST[continued] ## Retrying failed lifecycle policy steps [_retrying_failed_lifecycle_policy_steps] @@ -133,6 +138,7 @@ Once you fix the problem that put an index in the `ERROR` step, you might need t ```console POST /my-index-000001/_ilm/retry ``` +% TEST[skip:we can’t be sure the index is ready to be retried at this point] {{ilm-init}} subsequently attempts to re-run the step that failed. You can use the [{{ilm-init}} Explain API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-ilm-explain-lifecycle) to monitor the progress. diff --git a/troubleshoot/elasticsearch/red-yellow-cluster-status.md b/troubleshoot/elasticsearch/red-yellow-cluster-status.md index 9d9d9668e4..9ad503c433 100644 --- a/troubleshoot/elasticsearch/red-yellow-cluster-status.md +++ b/troubleshoot/elasticsearch/red-yellow-cluster-status.md @@ -54,6 +54,7 @@ GET _cluster/allocation/explain?filter_path=index,node_allocation_decisions.node "primary": false } ``` +% TEST[s/^/PUT my-index\n/] ## Fix a red or yellow cluster status [fix-red-yellow-cluster-status] @@ -102,6 +103,7 @@ GET my-index/_settings?flat_settings=true&include_defaults=true GET _cluster/settings?flat_settings=true&include_defaults=true ``` +% TEST[s/^/PUT my-index\n/] You can change the settings using the [update index settings](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-put-settings) and [cluster update settings](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-cluster-put-settings) APIs. @@ -119,6 +121,7 @@ PUT _settings "index.number_of_replicas": 1 } ``` +% TEST[s/^/PUT my-index\n/] ### Free up or increase disk space [fix-cluster-status-disk-space] diff --git a/troubleshoot/elasticsearch/remote-clusters.md b/troubleshoot/elasticsearch/remote-clusters.md index edd1d01404..de50500995 100644 --- a/troubleshoot/elasticsearch/remote-clusters.md +++ b/troubleshoot/elasticsearch/remote-clusters.md @@ -23,6 +23,8 @@ A successful call to the cluster settings update API for adding or updating remo ```console GET /_remote/info ``` +% TEST[skip:TODO] +% TEST[skip:TODO] The API should return `"connected" : true`. When using [API key authentication](../../deploy-manage/remote-clusters/remote-clusters-api-key.md), it should also return `"cluster_credentials": "::es_redacted::"`. @@ -42,6 +44,8 @@ The API should return `"connected" : true`. When using [API key authentication]( } } ``` +% TEST[skip:TODO] +% TEST[skip:TODO] 1. The remote cluster has connected successfully. 2. If present, indicates the remote cluster has connected using [API key authentication](../../deploy-manage/remote-clusters/remote-clusters-api-key.md) instead of [certificate based authentication](../../deploy-manage/remote-clusters/remote-clusters-cert.md). @@ -373,6 +377,7 @@ Request failures due to insufficient privileges result in API responses like: "reason": "action [indices:data/read/search] towards remote cluster is **unauthorized for user** [foo] with assigned roles [foo-role] authenticated by API key id [agZXJocBmA2beJfq2yKu] of user [elastic-admin] on indices [cd], this action is granted by the index privileges [read,all]" } ``` +% NOTCONSOLE This does not show up in any logs. @@ -398,6 +403,7 @@ This results in API responses like: "reason": "action [indices:data/read/search] towards remote cluster [my] is unauthorized for user [foo] with effective roles [] (assigned roles [foo-role] were not found) because **no remote indices privileges apply for the target cluster**" } ``` +% NOTCONSOLE #### Resolution [_resolution_9] diff --git a/troubleshoot/elasticsearch/repeated-snapshot-failures.md b/troubleshoot/elasticsearch/repeated-snapshot-failures.md index 022bb615b6..219a4e7602 100644 --- a/troubleshoot/elasticsearch/repeated-snapshot-failures.md +++ b/troubleshoot/elasticsearch/repeated-snapshot-failures.md @@ -105,6 +105,7 @@ In the event that snapshots are failing for other reasons check the logs on the ```console GET _slm/policy/ ``` +% TEST[skip:These policies do not exist] The response will look like this: @@ -151,6 +152,8 @@ The response will look like this: } } ``` +% TESTRESPONSE[skip:the result is for illustrating purposes only] +% TESTRESPONSE[skip:the result is for illustrating purposes only] 1. The affected snapshot lifecycle policy. 2. The information about the last failure for the policy. diff --git a/troubleshoot/elasticsearch/troubleshooting-searches.md b/troubleshoot/elasticsearch/troubleshooting-searches.md index c7f36787e6..625b5c0d3d 100644 --- a/troubleshoot/elasticsearch/troubleshooting-searches.md +++ b/troubleshoot/elasticsearch/troubleshooting-searches.md @@ -47,6 +47,7 @@ Use the [count API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/ ```console GET /my-index-000001/_count ``` +% TEST[continued] ::::{note} When getting no search results in {{kib}}, check that you have selected the correct data view and a valid time range. Also, ensure the data view has been configured with the correct time field. @@ -61,6 +62,7 @@ Querying a field that does not exist will not return any results. Use the [field ```console GET /my-index-000001/_field_caps?fields=my-field ``` +% TEST[continued] If the field does not exist, check the data ingestion process. The field may have a different name. @@ -97,6 +99,7 @@ A field’s capabilities are determined by its [mapping](../../manage-data/data- ```console GET /my-index-000001/_mappings ``` +% TEST[continued] If you query a `text` field, pay attention to the analyzer that may have been configured. You can use the [analyze API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-analyze) to check how a field’s analyzer processes values and query terms: @@ -107,6 +110,7 @@ GET /my-index-000001/_analyze "text" : "this is a test" } ``` +% TEST[continued] To change the mapping of an existing field, refer to [Changing the mapping of a field](../../manage-data/data-store/mapping.md#updating-field-mappings). @@ -125,6 +129,7 @@ GET /my-index-000001/_count } } ``` +% TEST[continued] If the field is aggregatable, you can use [aggregations](../../explore-analyze/query-filter/aggregations.md) to check the field’s values. For `keyword` fields, you can use a [terms aggregation](elasticsearch://reference/data-analysis/aggregations/search-aggregations-bucket-terms-aggregation.md) to retrieve the field’s most common values: @@ -142,6 +147,7 @@ GET /my-index-000001/_search?filter_path=aggregations } } ``` +% TEST[continued] For numeric fields, you can use the [stats aggregation](elasticsearch://reference/data-analysis/aggregations/search-aggregations-metrics-stats-aggregation.md) to get an idea of the field’s value distribution: @@ -157,6 +163,7 @@ GET my-index-000001/_search?filter_path=aggregations } } ``` +% TEST[continued] If the field does not return any values, check the data ingestion process. The field may have a different name. @@ -168,6 +175,7 @@ For time-series data, confirm there is non-filtered data within the attempted ti ```console GET my-index-000001/_search?sort=@timestamp:desc&size=1 ``` +% TEST[continued] ## Validate, explain, and profile queries [troubleshooting-searches-validate-explain-profile] @@ -189,6 +197,7 @@ GET /my-index-000001/_validate/query?rewrite=true } } ``` +% TEST[continued] Use the [explain API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-explain) to find out why a specific document matches or doesn’t match a query: @@ -200,6 +209,7 @@ GET /my-index-000001/_explain/0 } } ``` +% TEST[setup:messages] The [profile API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-profile.html) provides detailed timing information about a search request. For a visual representation of the results, use the [Search Profiler](../../explore-analyze/query-filter/tools/search-profiler.md) in {{kib}}. @@ -216,6 +226,7 @@ To troubleshoot queries in {{kib}}, select **Inspect** in the toolbar. Next, sel ```console GET /my-index-000001/_settings ``` +% TEST[continued] You can update dynamic index settings with the [update index settings API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-put-settings). [Changing dynamic index settings for a data stream](../../manage-data/data-store/data-streams/modify-data-stream.md#change-dynamic-index-setting-for-a-data-stream) requires changing the index template used by the data stream. diff --git a/troubleshoot/elasticsearch/troubleshooting-shards-capacity-issues.md b/troubleshoot/elasticsearch/troubleshooting-shards-capacity-issues.md index 739b64469d..961e02a3f0 100644 --- a/troubleshoot/elasticsearch/troubleshooting-shards-capacity-issues.md +++ b/troubleshoot/elasticsearch/troubleshooting-shards-capacity-issues.md @@ -133,6 +133,7 @@ Check the current status of the cluster according the shards capacity indicator: ```console GET _health_report/shards_capacity ``` +% TESTRESPONSE[skip:the result is for illustrating purposes only] The response will look like this: @@ -161,6 +162,7 @@ The response will look like this: } } ``` +% TESTRESPONSE[skip:the result is for illustrating purposes only] 1. Current value of the setting `cluster.max_shards_per_node` 2. Current number of open shards across the cluster @@ -204,6 +206,7 @@ The response will look like this: } } ``` +% TESTRESPONSE[skip:the result is for illustrating purposes only] When a long-term solution is in place, we recommend you reset the `cluster.max_shards_per_node` limit. @@ -368,6 +371,7 @@ GET _health_report/shards_capacity } } ``` +% TESTRESPONSE[skip:the result is for illustrating purposes only] 1. Current value of the setting `cluster.max_shards_per_node.frozen`. 2. Current number of open shards used by frozen nodes across the cluster. @@ -411,6 +415,7 @@ The response will look like this: } } ``` +% TESTRESPONSE[skip:the result is for illustrating purposes only] When a long-term solution is in place, we recommend you reset the `cluster.max_shards_per_node.frozen` limit. diff --git a/troubleshoot/elasticsearch/troubleshooting-unbalanced-cluster.md b/troubleshoot/elasticsearch/troubleshooting-unbalanced-cluster.md index 433732719e..45d738deb3 100644 --- a/troubleshoot/elasticsearch/troubleshooting-unbalanced-cluster.md +++ b/troubleshoot/elasticsearch/troubleshooting-unbalanced-cluster.md @@ -29,6 +29,7 @@ Use the [cat allocation command](https://www.elastic.co/docs/api/doc/elasticsear ```console GET /_cat/allocation?v ``` +% TEST[s/^/PUT test\n{"settings": {"number_of_replicas": 0}}\n/] The API returns the following response: @@ -36,6 +37,8 @@ The API returns the following response: shards shards.undesired write_load.forecast disk.indices.forecast disk.indices disk.used disk.avail disk.total disk.percent host ip node node.role 1 0 0.0 260b 260b 47.3gb 43.4gb 100.7gb 46 127.0.0.1 127.0.0.1 CSUXak2 himrst ``` +% TESTRESPONSE[s/\d+(.\d+)?[tgmk]?b/\d+(\.\d+)?[tgmk]?b/ s/46/\d+/] +% TESTRESPONSE[s/CSUXak2 himrst/.+/ non_json] This response contains the following information that influences balancing: diff --git a/troubleshoot/elasticsearch/watcher-troubleshooting.md b/troubleshoot/elasticsearch/watcher-troubleshooting.md index a686eba982..0ae4c06e3c 100644 --- a/troubleshoot/elasticsearch/watcher-troubleshooting.md +++ b/troubleshoot/elasticsearch/watcher-troubleshooting.md @@ -14,6 +14,7 @@ If you get the *Dynamic Mapping is Disabled* error when you try to add a watch, ```console GET .watches/_mapping ``` +% TEST[skip:deprecation warning] If the index mappings are missing, follow these steps to restore the correct mappings: From c18708ea812834312f91ca7ca37a586f057d1561 Mon Sep 17 00:00:00 2001 From: Colleen McGinnis Date: Thu, 6 Mar 2025 15:29:30 -0600 Subject: [PATCH 2/4] clean up --- .../autoscaling/autoscaling-deciders.md | 3 ++ .../google-cloud-storage-repository.md | 2 +- .../snapshot-and-restore/restore-snapshot.md | 4 +- ...trolling-access-at-document-field-level.md | 1 + .../cluster-or-deployment-auth/jwt.md | 5 +++ .../cluster-or-deployment-auth/native.md | 1 + .../watcher/watch-cluster-status.md | 2 +- .../data-streams/run-downsampling-manually.md | 3 +- ...ownsampling-using-data-stream-lifecycle.md | 4 +- .../data-streams/run-downsampling-with-ilm.md | 4 +- manage-data/data-store/index-basics.md | 1 + .../ignore-missing-component-templates.md | 3 ++ ...ed-data-stream-to-data-stream-lifecycle.md | 8 ++-- .../tutorial-update-existing-data-stream.md | 8 ++-- manage-data/lifecycle/data-tiers.md | 1 + ...-index-allocation-filters-to-node-roles.md | 2 +- solutions/search/cross-cluster-search.md | 2 +- solutions/search/retrievers-examples.md | 32 +++++++++++++++ .../search-application-api.md | 1 - solutions/search/search-templates.md | 40 +++++++++---------- solutions/search/vector/knn.md | 19 +++++++++ .../elasticsearch/increase-tier-capacity.md | 4 +- troubleshoot/elasticsearch/remote-clusters.md | 4 +- .../repeated-snapshot-failures.md | 1 - troubleshoot/elasticsearch/start-ilm.md | 6 +++ 25 files changed, 116 insertions(+), 45 deletions(-) diff --git a/deploy-manage/autoscaling/autoscaling-deciders.md b/deploy-manage/autoscaling/autoscaling-deciders.md index 90f94fd949..740a7e221f 100644 --- a/deploy-manage/autoscaling/autoscaling-deciders.md +++ b/deploy-manage/autoscaling/autoscaling-deciders.md @@ -80,6 +80,7 @@ PUT /_autoscaling/policy/my_autoscaling_policy } } ``` +% TEST The API returns the following result: @@ -157,6 +158,7 @@ PUT /_autoscaling/policy/my_autoscaling_policy } } ``` +% TEST The API returns the following result: @@ -213,6 +215,7 @@ PUT /_autoscaling/policy/my_autoscaling_policy } } ``` +% TEST The API returns the following result: diff --git a/deploy-manage/tools/snapshot-and-restore/google-cloud-storage-repository.md b/deploy-manage/tools/snapshot-and-restore/google-cloud-storage-repository.md index cfb98d9926..cb5cab3328 100644 --- a/deploy-manage/tools/snapshot-and-restore/google-cloud-storage-repository.md +++ b/deploy-manage/tools/snapshot-and-restore/google-cloud-storage-repository.md @@ -91,7 +91,6 @@ PUT _snapshot/my_gcs_repository } ``` % TEST[skip:we don’t have gcs setup while testing this] -% TEST[skip:we don’t have gcs setup while testing this] The `credentials_file` settings are [reloadable](../../security/secure-settings.md#reloadable-secure-settings). You can define these settings before the node is started, or call the [Nodes reload secure settings API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-nodes-reload-secure-settings) after the settings are defined to apply them to a running node. @@ -120,6 +119,7 @@ PUT _snapshot/my_gcs_repository } } ``` +% TEST[skip:we don’t have gcs setup while testing this] Some settings are sensitive and must be stored in the [Elasticsearch keystore](../../security/secure-settings.md). This is the case for the service account file: diff --git a/deploy-manage/tools/snapshot-and-restore/restore-snapshot.md b/deploy-manage/tools/snapshot-and-restore/restore-snapshot.md index 6e4ac29560..17ac58d95f 100644 --- a/deploy-manage/tools/snapshot-and-restore/restore-snapshot.md +++ b/deploy-manage/tools/snapshot-and-restore/restore-snapshot.md @@ -93,7 +93,6 @@ DELETE my-index DELETE _data_stream/logs-my_app-default ``` % TEST[setup:setup-snapshots] -% TEST[setup:setup-snapshots] In the restore request, explicitly specify any indices and data streams to restore. ```console @@ -118,7 +117,8 @@ POST _snapshot/my_repository/my_snapshot_2099.05.06/_restore { "indices": "my-index,logs-my_app-default", "rename_pattern": "(.+)", - "rename_replacement": "restored-$1" + "rename_replacement": "restored- +If the rename options produce two or more indices or data streams with the same name, the restore operation fails." } ``` % TEST[setup:setup-snapshots] diff --git a/deploy-manage/users-roles/cluster-or-deployment-auth/controlling-access-at-document-field-level.md b/deploy-manage/users-roles/cluster-or-deployment-auth/controlling-access-at-document-field-level.md index 5c52dbfa15..831deb1627 100644 --- a/deploy-manage/users-roles/cluster-or-deployment-auth/controlling-access-at-document-field-level.md +++ b/deploy-manage/users-roles/cluster-or-deployment-auth/controlling-access-at-document-field-level.md @@ -255,6 +255,7 @@ Use the dot notations to refer to nested fields in more complex documents. For e } } ``` +% NOTCONSOLE The following role definition enables only read access to the customer `handle` field: diff --git a/deploy-manage/users-roles/cluster-or-deployment-auth/jwt.md b/deploy-manage/users-roles/cluster-or-deployment-auth/jwt.md index 3202b68885..21806d84bb 100644 --- a/deploy-manage/users-roles/cluster-or-deployment-auth/jwt.md +++ b/deploy-manage/users-roles/cluster-or-deployment-auth/jwt.md @@ -205,6 +205,7 @@ Header: {"typ":"JWT","alg":"HS256"} Claims: {"aud":"aud8","sub":"security_test_user","iss":"iss8","exp":4070908800,"iat":946684800} Signature: UnnFmsoFKfNmKMsVoDQmKI_3-j95PCaKdgqqau3jPMY ``` +% NOTCONSOLE This example illustrates a partial decoding of a JWT. The validity period is from 2000 to 2099 (inclusive), as defined by the issue time (`iat`) and expiration time (`exp`). JWTs typically have a validity period shorter than 100 years, such as 1-2 hours or 1-7 days, not an entire human life. @@ -396,6 +397,7 @@ $$$jwt-auth-shared-secret-scheme-example$$$ ```sh curl -s -X GET -H "Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOlsiZXMwMSIsImVzMDIiLCJlczAzIl0sInN1YiI6InVzZXIyIiwiaXNzIjoibXktaXNzdWVyIiwiZXhwIjo0MDcwOTA4ODAwLCJpYXQiOjk0NjY4NDgwMCwiZW1haWwiOiJ1c2VyMkBzb21ldGhpbmcuZXhhbXBsZS5jb20ifQ.UgO_9w--EoRyUKcWM5xh9SimTfMzl1aVu6ZBsRWhxQA" -H "ES-Client-Authentication: sharedsecret test-secret" https://localhost:9200/_security/_authenticate ``` +% NOTCONSOLE The response includes the user who submitted the request (`user2`), including the `jwt_role1` role that you mapped to this user in the JWT realm: @@ -412,6 +414,7 @@ If you want to specify a request as the `run_as` user, include the `es-security- ```sh curl -s -X GET -H "Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOlsiZXMwMSIsImVzMDIiLCJlczAzIl0sInN1YiI6InVzZXIyIiwiaXNzIjoibXktaXNzdWVyIiwiZXhwIjo0MDcwOTA4ODAwLCJpYXQiOjk0NjY4NDgwMCwiZW1haWwiOiJ1c2VyMkBzb21ldGhpbmcuZXhhbXBsZS5jb20ifQ.UgO_9w--EoRyUKcWM5xh9SimTfMzl1aVu6ZBsRWhxQA" -H "ES-Client-Authentication: sharedsecret test-secret" -H "es-security-runas-user: user123_runas" https://localhost:9200/_security/_authenticate ``` +% NOTCONSOLE In the response, you’ll see that the `user123_runas` user submitted the request, and {{es}} used the `jwt_role1` role: @@ -530,12 +533,14 @@ The following header settings are for an {{es}} client. Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJpc3M4IiwiYXVkIjoiYXVkOCIsInN1YiI6InNlY3VyaXR5X3Rlc3RfdXNlciIsImV4cCI6NDA3MDkwODgwMCwiaWF0Ijo5NDY2ODQ4MDB9.UnnFmsoFKfNmKMsVoDQmKI_3-j95PCaKdgqqau3jPMY ES-Client-Authentication: SharedSecret client-shared-secret-string ``` +% NOTCONSOLE You can use this header in a `curl` request to make an authenticated call to {{es}}. Both the bearer token and the client authorization token must be specified as separate headers with the `-H` option: ```sh curl -s -X GET -H "Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJpc3M4IiwiYXVkIjoiYXVkOCIsInN1YiI6InNlY3VyaXR5X3Rlc3RfdXNlciIsImV4cCI6NDA3MDkwODgwMCwiaWF0Ijo5NDY2ODQ4MDB9.UnnFmsoFKfNmKMsVoDQmKI_3-j95PCaKdgqqau3jPMY" -H "ES-Client-Authentication: SharedSecret client-shared-secret-string" https://localhost:9200/_security/_authenticate ``` +% NOTCONSOLE If you used role mapping in the JWT realm, the response includes the user’s `username`, their `roles`, metadata about the user, and the details about the JWT realm itself. diff --git a/deploy-manage/users-roles/cluster-or-deployment-auth/native.md b/deploy-manage/users-roles/cluster-or-deployment-auth/native.md index 0f73cdfbe2..9a8dde4250 100644 --- a/deploy-manage/users-roles/cluster-or-deployment-auth/native.md +++ b/deploy-manage/users-roles/cluster-or-deployment-auth/native.md @@ -90,6 +90,7 @@ POST /_security/user/user1/_password "password" : "new-test-password" } ``` +% TEST[continued] For more information and examples, see [Users](https://www.elastic.co/docs/api/doc/elasticsearch/group/endpoint-security). diff --git a/explore-analyze/alerts-cases/watcher/watch-cluster-status.md b/explore-analyze/alerts-cases/watcher/watch-cluster-status.md index 499bf85f27..efadf04388 100644 --- a/explore-analyze/alerts-cases/watcher/watch-cluster-status.md +++ b/explore-analyze/alerts-cases/watcher/watch-cluster-status.md @@ -146,7 +146,6 @@ GET .watcher-history*/_search?pretty } ``` % TEST[continued] -% TEST[continued] ## Take action [health-take-action] @@ -223,6 +222,7 @@ GET .watcher-history*/_search?pretty } } ``` +% TEST[continued] ## Delete the watch [health-delete] diff --git a/manage-data/data-store/data-streams/run-downsampling-manually.md b/manage-data/data-store/data-streams/run-downsampling-manually.md index 43df992f50..960d66adf2 100644 --- a/manage-data/data-store/data-streams/run-downsampling-manually.md +++ b/manage-data/data-store/data-streams/run-downsampling-manually.md @@ -257,7 +257,6 @@ You can use the search API to check if the documents have been indexed correctly ```console GET /my-data-stream/_search ``` -% TEST[skip:Because we’ve skipped the previous steps] % TEST[continued] Run the following aggregation on the data to calculate some interesting statistics: @@ -423,6 +422,7 @@ Re-run the earlier search query (note that when querying downsampled indices the ```console GET /my-data-stream/_search ``` +% TEST[skip:Because we’ve skipped the previous steps] The TSDS with the new downsampled backing index contains just one document. For counters, this document would only have the last value. For gauges, the field type is now `aggregate_metric_double`. You see the `min`, `max`, `sum`, and `value_count` statistics based off of the original sampled metrics: @@ -575,6 +575,7 @@ GET /my-data-stream/_search } } ``` +% TEST[continued] This example demonstrates how downsampling can dramatically reduce the number of documents stored for time series data, within whatever time boundaries you choose. It’s also possible to perform downsampling on already downsampled data, to further reduce storage and associated costs, as the time series data ages and the data resolution becomes less critical. diff --git a/manage-data/data-store/data-streams/run-downsampling-using-data-stream-lifecycle.md b/manage-data/data-store/data-streams/run-downsampling-using-data-stream-lifecycle.md index 7f285394cb..5cf053d3be 100644 --- a/manage-data/data-store/data-streams/run-downsampling-using-data-stream-lifecycle.md +++ b/manage-data/data-store/data-streams/run-downsampling-using-data-stream-lifecycle.md @@ -230,7 +230,6 @@ Now that you’ve created and added documents to the data stream, check to confi GET _data_stream ``` % TEST[skip: temporal_ranges and index names won’t match] -% TEST[skip: temporal_ranges and index names won’t match] If the data stream lifecycle policy has not yet been applied, your results will be like the following. Note the original `index_name`: `.ds-datastream-2024.04.29-000001`. @@ -287,7 +286,6 @@ Next, run a search query: ```console GET datastream/_search ``` -% TEST[continued] % TEST[skip: timestamp values won’t match] The query returns your ten newly added documents. @@ -332,6 +330,7 @@ By default, data stream lifecycle actions are executed every five minutes. Downs ```console GET _data_stream ``` +% TEST[skip: temporal_ranges and index names won’t match] After the data stream lifecycle action was executed, original `.ds-datastream-2024.04.29-000001` index is replaced with a new, downsampled index, in this case `downsample-1h-.ds-datastream-2024.04.29-000001`. @@ -365,6 +364,7 @@ Run a search query on the datastream (note that when querying downsampled indice ```console GET datastream/_search ``` +% TEST[continued] The new downsampled index contains just one document that includes the `min`, `max`, `sum`, and `value_count` statistics based off of the original sampled metrics. diff --git a/manage-data/data-store/data-streams/run-downsampling-with-ilm.md b/manage-data/data-store/data-streams/run-downsampling-with-ilm.md index 4b49144246..2e92136af5 100644 --- a/manage-data/data-store/data-streams/run-downsampling-with-ilm.md +++ b/manage-data/data-store/data-streams/run-downsampling-with-ilm.md @@ -254,7 +254,6 @@ Now that you’ve created and added documents to the data stream, check to confi GET _data_stream ``` % TEST[skip: The @timestamp value won’t match an accepted range in the TSDS] -% TEST[skip: The @timestamp value won’t match an accepted range in the TSDS] If the ILM policy has not yet been applied, your results will be like the following. Note the original `index_name`: `.ds-datastream--000001`. @@ -305,7 +304,6 @@ Next, run a search query: ```console GET datastream/_search ``` -% TEST[continued] % TEST[skip: The @timestamp value won’t match an accepted range in the TSDS] The query returns your ten newly added documents. @@ -335,6 +333,7 @@ By default, index lifecycle management checks every ten minutes for indices that ```console GET _data_stream ``` +% TEST[skip: The @timestamp value won’t match an accepted range in the TSDS] After the ILM policy has taken effect, the original `.ds-datastream-2022.08.26-000001` index is replaced with a new, downsampled index, in this case `downsample-6tkn-.ds-datastream-2022.08.26-000001`. @@ -366,6 +365,7 @@ Run a search query on the datastream (note that when querying downsampled indice ```console GET datastream/_search ``` +% TEST[continued] The new downsampled index contains just one document that includes the `min`, `max`, `sum`, and `value_count` statistics based off of the original sampled metrics. diff --git a/manage-data/data-store/index-basics.md b/manage-data/data-store/index-basics.md index 2dbeefb868..552843b3b8 100644 --- a/manage-data/data-store/index-basics.md +++ b/manage-data/data-store/index-basics.md @@ -52,6 +52,7 @@ A simple {{es}} document might look like this: } } ``` +% NOTCONSOLE ### Metadata fields [elasticsearch-intro-documents-fields-data-metadata] diff --git a/manage-data/data-store/templates/ignore-missing-component-templates.md b/manage-data/data-store/templates/ignore-missing-component-templates.md index cfc1bf2b20..1940fba0c6 100644 --- a/manage-data/data-store/templates/ignore-missing-component-templates.md +++ b/manage-data/data-store/templates/ignore-missing-component-templates.md @@ -69,12 +69,14 @@ PUT _index_template/logs-foo "priority": 500 } ``` +% TEST[continued] A data stream can be created based on this template: ```console PUT _data_stream/logs-foo-bar ``` +% TEST[continued] The mappings of the data stream will contain the `host.name` field. The missing component template can be added later: @@ -93,6 +95,7 @@ PUT _component_template/logs-foo_component2 } } ``` +% TEST[continued] This will not have an immediate effect on the data stream. The mapping `host.ip` will only show up in the data stream mappings when the data stream is rolled over automatically next time or a manual rollover is triggered: diff --git a/manage-data/lifecycle/data-stream/tutorial-migrate-ilm-managed-data-stream-to-data-stream-lifecycle.md b/manage-data/lifecycle/data-stream/tutorial-migrate-ilm-managed-data-stream-to-data-stream-lifecycle.md index 580a1ed3b4..8d6108390f 100644 --- a/manage-data/lifecycle/data-stream/tutorial-migrate-ilm-managed-data-stream-to-data-stream-lifecycle.md +++ b/manage-data/lifecycle/data-stream/tutorial-migrate-ilm-managed-data-stream-to-data-stream-lifecycle.md @@ -80,7 +80,6 @@ POST dsl-data-stream/_doc? POST dsl-data-stream/_rollover ``` % TEST[continued] -% TEST[continued] We’ll use the [GET _data_stream](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-get-data-stream) API to inspect the state of the data stream: @@ -88,9 +87,6 @@ We’ll use the [GET _data_stream](https://www.elastic.co/docs/api/doc/elasticse GET _data_stream/dsl-data-stream ``` % TEST[continued] -% TEST[continued] -% TEST[continued] -% TEST[continued] Inspecting the response we’ll see that both backing indices are managed by {{ilm-init}} and that the next generation index will also be managed by {{ilm-init}}: @@ -204,6 +200,7 @@ We can inspect the data stream to check that the next generation will indeed be ```console GET _data_stream/dsl-data-stream ``` +% TEST[continued] ```console-result { @@ -267,10 +264,12 @@ We’ll now rollover the data stream to see the new generation index being manag ```console POST dsl-data-stream/_rollover ``` +% TEST[continued] ```console GET _data_stream/dsl-data-stream ``` +% TEST[continued] ```console-result { @@ -365,6 +364,7 @@ PUT _data_stream/dsl-data-stream/_lifecycle ```console GET _data_stream/dsl-data-stream ``` +% TEST[continued] ```console-result { diff --git a/manage-data/lifecycle/data-stream/tutorial-update-existing-data-stream.md b/manage-data/lifecycle/data-stream/tutorial-update-existing-data-stream.md index 7589edce6f..50339b8dbc 100644 --- a/manage-data/lifecycle/data-stream/tutorial-update-existing-data-stream.md +++ b/manage-data/lifecycle/data-stream/tutorial-update-existing-data-stream.md @@ -14,7 +14,7 @@ To update the lifecycle of an existing data stream you do the following actions: 2. [Remove lifecycle for a data stream](#delete-lifecycle) -## Set a data stream’s lifecycle [set-lifecycle] +## Set a data stream’s lifecycle [set-lifecycle] To add or to change the retention period of your data stream you can use the [PUT lifecycle API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-put-data-lifecycle). @@ -45,8 +45,6 @@ The changes in the lifecycle are applied on all backing indices of the data stre GET .ds-my-data-stream-*/_lifecycle/explain ``` % TEST[continued] -% TEST[teardown:data_stream_cleanup] -% TEST[continued] The response will look like: @@ -95,7 +93,7 @@ The response will look like: -## Remove lifecycle for a data stream [delete-lifecycle] +## Remove lifecycle for a data stream [delete-lifecycle] To remove the lifecycle of a data stream you can use the [delete lifecycle API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-delete-data-lifecycle). As consequence, the maintenance operations that were applied by the lifecycle will no longer be applied to the data stream and all its backing indices. For example: @@ -109,6 +107,8 @@ You can then use the [explain API](https://www.elastic.co/docs/api/doc/elasticse ```console GET .ds-my-data-stream-*/_lifecycle/explain ``` +% TEST[continued] +% TEST[teardown:data_stream_cleanup] ```console-result { diff --git a/manage-data/lifecycle/data-tiers.md b/manage-data/lifecycle/data-tiers.md index 470789937f..c64c7e57bd 100644 --- a/manage-data/lifecycle/data-tiers.md +++ b/manage-data/lifecycle/data-tiers.md @@ -476,6 +476,7 @@ You can check an existing index’s data tier preference by [polling its setting ```console GET /my-index-000001/_settings?filter_path=*.settings.index.routing.allocation.include._tier_preference ``` +% TEST[setup:my_index] ### Troubleshooting [data-tier-allocation-troubleshooting] diff --git a/manage-data/lifecycle/index-lifecycle-management/migrate-index-allocation-filters-to-node-roles.md b/manage-data/lifecycle/index-lifecycle-management/migrate-index-allocation-filters-to-node-roles.md index 32387c7018..d4601846de 100644 --- a/manage-data/lifecycle/index-lifecycle-management/migrate-index-allocation-filters-to-node-roles.md +++ b/manage-data/lifecycle/index-lifecycle-management/migrate-index-allocation-filters-to-node-roles.md @@ -127,7 +127,6 @@ PUT my-index/_settings } ``` % TEST[continued] -% TEST[continued] If an index is already in the cold phase, include the cold, warm, and hot tiers. @@ -151,6 +150,7 @@ PUT my-index/_settings "index.routing.allocation.include._tier_preference": "data_warm,data_hot" } ``` +% TEST[continued] This situation can occur in a system that defaults to data tiers when, e.g., an ILM policy that uses node attributes is restored and transitions the managed indices from the hot phase into the warm phase. In this case the node attribute configuration indicates the correct tier where the index should be allocated. diff --git a/solutions/search/cross-cluster-search.md b/solutions/search/cross-cluster-search.md index 901db28188..93541103cc 100644 --- a/solutions/search/cross-cluster-search.md +++ b/solutions/search/cross-cluster-search.md @@ -445,7 +445,6 @@ If you set `ccs_minimize_roundtrips=true`, then you will also see partial result GET /_async_search/FklQYndoTDJ2VEFlMEVBTzFJMGhJVFEaLVlKYndBWWZSMUdicUc4WVlEaFl4ZzoxNTU= ``` % TEST[continued s/FklQYndoTDJ2VEFlMEVBTzFJMGhJVFEaLVlKYndBWWZSMUdicUc4WVlEaFl4ZzoxNTU=/${body.id}/] -% TEST[continued s/FklQYndoTDJ2VEFlMEVBTzFJMGhJVFEaLVlKYndBWWZSMUdicUc4WVlEaFl4ZzoxNTU=/${body.id}/] Response: @@ -527,6 +526,7 @@ After searches on all the clusters have completed, querying the [get async searc ```console GET /_async_search/FklQYndoTDJ2VEFlMEVBTzFJMGhJVFEaLVlKYndBWWZSMUdicUc4WVlEaFl4ZzoxNTU= ``` +% TEST[continued s/FklQYndoTDJ2VEFlMEVBTzFJMGhJVFEaLVlKYndBWWZSMUdicUc4WVlEaFl4ZzoxNTU=/${body.id}/] Response: diff --git a/solutions/search/retrievers-examples.md b/solutions/search/retrievers-examples.md index fb9480a441..5bdf1def0c 100644 --- a/solutions/search/retrievers-examples.md +++ b/solutions/search/retrievers-examples.md @@ -142,6 +142,7 @@ GET /retrievers_example/_search "_source": false } ``` +% TEST This returns the following response based on the final rrf score for each result. @@ -182,6 +183,7 @@ This returns the following response based on the final rrf score for each result } } ``` +% TESTRESPONSE[s/"took": 42/"took": $body.took/] :::: @@ -199,6 +201,7 @@ So, let’s now specify the `linear` retriever whose final score is computed as score = weight(standard) * score(standard) + weight(knn) * score(knn) score = 2 * score(standard) + 1.5 * score(knn) ``` +% NOTCONSOLE ```console GET /retrievers_example/_search @@ -243,6 +246,7 @@ GET /retrievers_example/_search "_source": false } ``` +% TEST[continued] This returns the following response based on the normalized weighted score for each result. @@ -283,6 +287,11 @@ This returns the following response based on the normalized weighted score for e } } ``` +% TESTRESPONSE[s/"took": 42/"took": $body.took/] +% TESTRESPONSE[s/"max_score": -1/"max_score": $body.hits.max_score/] +% TESTRESPONSE[s/"_score": -1/"_score": $body.hits.hits.0._score/] +% TESTRESPONSE[s/"_score": -2/"_score": $body.hits.hits.1._score/] +% TESTRESPONSE[s/"_score": -3/"_score": $body.hits.hits.2._score/] :::: @@ -349,6 +358,7 @@ GET /retrievers_example/_search "_source": false } ``` +% TEST[continued] Which would return the following results: @@ -394,6 +404,12 @@ Which would return the following results: } } ``` +% TESTRESPONSE[s/"took": 42/"took": $body.took/] +% TESTRESPONSE[s/"max_score": -1/"max_score": $body.hits.max_score/] +% TESTRESPONSE[s/"_score": -1/"_score": $body.hits.hits.0._score/] +% TESTRESPONSE[s/"_score": -2/"_score": $body.hits.hits.1._score/] +% TESTRESPONSE[s/"_score": -3/"_score": $body.hits.hits.2._score/] +% TESTRESPONSE[s/"_score": -4/"_score": $body.hits.hits.3._score/] :::: @@ -449,6 +465,7 @@ GET /retrievers_example/_search "_source": false } ``` +% TEST[continued] This returns the following response with collapsed results. @@ -544,6 +561,7 @@ This returns the following response with collapsed results. } } ``` +% TESTRESPONSE[s/"took": 42/"took": $body.took/] :::: @@ -597,6 +615,7 @@ GET /retrievers_example/_search "_source": false } ``` +% TEST[continued] This would highlight the `text` field, based on the matches produced by the `standard` retriever. The highlighted snippets would then be included in the response as usual, i.e. under each search hit. @@ -647,6 +666,7 @@ This would highlight the `text` field, based on the matches produced by the `sta } } ``` +% TESTRESPONSE[s/"took": 42/"took": $body.took/] :::: @@ -747,6 +767,7 @@ POST /retrievers_example_nested/_doc/3 POST /retrievers_example_nested/_refresh ``` +% TEST[continued] Now we can run an `rrf` retriever query and also compute [inner hits](elasticsearch://reference/elasticsearch/rest-apis/retrieve-inner-hits.md) for the `nested_field.nested_vector` field, based on the `knn` query specified. @@ -802,6 +823,7 @@ GET /retrievers_example_nested/_search ] } ``` +% TEST[continued] This would propagate the `inner_hits` defined for the `knn` query to the `rrf` retriever, and compute inner hits for `rrf`'s top results. @@ -967,6 +989,7 @@ This would propagate the `inner_hits` defined for the `knn` query to the `rrf` r } } ``` +% TESTRESPONSE[s/"took": 42/"took": $body.took/] :::: @@ -1019,6 +1042,7 @@ GET retrievers_example/_search } } ``` +% TEST[continued] ::::{dropdown} Example response ```console-result @@ -1102,6 +1126,7 @@ GET retrievers_example/_search } } ``` +% TESTRESPONSE[s/"took": 42/"took": $body.took/] :::: @@ -1166,6 +1191,7 @@ GET /retrievers_example/_search "explain": true } ``` +% TEST[continued] The output of which, albeit a bit verbose, will provide all the necessary info to assist in debugging and reason with ranking. @@ -1264,6 +1290,10 @@ The output of which, albeit a bit verbose, will provide all the necessary info t } } ``` +% TESTRESPONSE[s/"took": 42/"took": $body.took/] +% TESTRESPONSE[s/.../$body.hits.hits.0._explanation.details.1.details.0.details.0.details.0.details.0.details.0/] +% TESTRESPONSE[s/***/$body.hits.hits.0._explanation.details.1.details.0.details.0.details.0.details.1.details.0/] +% TESTRESPONSE[s/jnrdZFKS3abUgWVsVdj2Vg/$body.hits.hits.0._node/] :::: @@ -1285,6 +1315,7 @@ PUT _inference/rerank/my-rerank-model } } ``` +% TEST[skip: no_access_to_ml] Let’s start by reranking the results of the `rrf` retriever in our previous example. @@ -1419,6 +1450,7 @@ GET retrievers_example/_search "_source": false } ``` +% TEST[skip: no_access_to_ml] Note that our example applies two reranking steps. First, we rerank the top 100 documents from the `knn` search using the `my-rerank-model` reranker. Then we pick the top 10 results and rerank them using the more fine-grained `my-other-more-expensive-rerank-model`. diff --git a/solutions/search/search-applications/search-application-api.md b/solutions/search/search-applications/search-application-api.md index 935f4bf076..75694eff22 100644 --- a/solutions/search/search-applications/search-application-api.md +++ b/solutions/search/search-applications/search-application-api.md @@ -175,7 +175,6 @@ When you actually perform a search with no parameters, it will execute the under POST _application/search_application/my_search_application/_search ``` % TEST[continued] -% TEST[continued] Searching with the `query_string` and/or `default_field` parameters will perform a [`query_string`](elasticsearch://reference/query-languages/query-dsl-query-string-query.md) query. diff --git a/solutions/search/search-templates.md b/solutions/search/search-templates.md index 1beb82fd84..d51e84e382 100644 --- a/solutions/search/search-templates.md +++ b/solutions/search/search-templates.md @@ -15,7 +15,7 @@ If you use {{es}} as a search backend, you can pass user input from a search bar If you use {{es}} for a custom application, search templates let you change your searches without modifying your app’s code. -### Create a search template [create-search-template] +### Create a search template [create-search-template] To create or update a search template, use the [create stored script API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-put-script). @@ -46,7 +46,7 @@ PUT _scripts/my-search-template {{es}} stores search templates as Mustache [scripts](../../explore-analyze/scripting.md) in the cluster state. {{es}} compiles search templates in the `template` script context. Settings that limit or disable scripts also affect search templates. -### Validate a search template [validate-search-template] +### Validate a search template [validate-search-template] $$$_validating_templates$$$ To test a template with different `params`, use the [render search template API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-render-search-template). @@ -63,7 +63,6 @@ POST _render/template } ``` % TEST[continued] -% TEST[continued] When rendered, the template outputs a [search request body](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search). @@ -105,7 +104,7 @@ POST _render/template % TEST[continued] -### Run a templated search [run-templated-search] +### Run a templated search [run-templated-search] To run a search with a search template, use the [search template API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search-template). You can specify different `params` with each request. @@ -156,7 +155,7 @@ The response uses the same properties as the [search API](https://www.elastic.co % TESTRESPONSE[s/"took": 36/"took": "$body.took"/] -### Run multiple templated searches [run-multiple-templated-searches] +### Run multiple templated searches [run-multiple-templated-searches] To run multiple templated searches with a single request, use the [multi search template API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-msearch-template). These requests often have less overhead and faster speeds than multiple individual searches. @@ -171,7 +170,7 @@ GET my-index/_msearch/template % TEST[s/my-other-search-template/my-search-template/] -### Get search templates [get-search-templates] +### Get search templates [get-search-templates] To retrieve a search template, use the [get stored script API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-get-script). @@ -188,7 +187,7 @@ GET _cluster/state/metadata?pretty&filter_path=metadata.stored_scripts % TEST[continued] -### Delete a search template [delete-search-template] +### Delete a search template [delete-search-template] To delete a search template, use the [delete stored script API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-delete-script). @@ -198,7 +197,7 @@ DELETE _scripts/my-search-template % TEST[continued] -### Set default values [search-template-set-default-values] +### Set default values [search-template-set-default-values] To set a default value for a variable, use the following syntax: @@ -227,7 +226,7 @@ POST _render/template ``` -### URL encode strings [search-template-url-encode-strings] +### URL encode strings [search-template-url-encode-strings] Use the `{{#url}}` function to URL encode a string. @@ -263,7 +262,7 @@ The template renders as: ``` -### Concatenate values [search-template-concatenate-values] +### Concatenate values [search-template-concatenate-values] Use the `{{#join}}` function to concatenate array values as a comma-delimited string. For example, the following template concatenates two email addresses. @@ -342,7 +341,7 @@ The template renders as: ``` -### Convert to JSON [search-template-convert-json] +### Convert to JSON [search-template-convert-json] Use the `{{#toJson}}` function to convert a variable value to its JSON representation. @@ -453,7 +452,7 @@ The template renders as: ``` -### Use conditions [search-template-use-conditions] +### Use conditions [search-template-use-conditions] To create if conditions, use the following syntax: @@ -559,7 +558,7 @@ POST _render/template The mustache templating language defines various tag types you can use within templates. The following sections describe some of these tag types and provide examples of using them in {{es}} search templates. -### Mustache variables [search-template-mustache-variable] +### Mustache variables [search-template-mustache-variable] Mustache tags are typically enclosed in double curly brackets. A mustache variable: `{{my-variable}}` is a type of mustache tag. When you run a templated search, {{es}} replaces these variables with values from `params`. @@ -596,6 +595,7 @@ POST _render/template } } ``` +% TEST[continued] When rendered, the `{{query_string}}` in `message` is replaced with `hello world` passed in `params`. @@ -646,7 +646,7 @@ When rendered, template outputs as: ``` -### Sections [search-template-sections] +### Sections [search-template-sections] Sections are also a type of Mustache tags. You can use `sections` in your search template with a nested or unnested object. A section begins with `{{#my-section-variable}}` and ends with `{{/my-section-variable}}`. @@ -695,7 +695,7 @@ The template renders as: ``` -#### Lists [search-template-lists] +#### Lists [search-template-lists] You can pass a list of objects and loop over each item in your search template. @@ -754,7 +754,7 @@ When rendered, template outputs: } ``` -::::{note} +::::{note} The above will cause a trailing comma issue, which causes invalid JSON. A workaround would be to include an [inverted section](#search-template-inverted-section) and adding a variable to make sure it’s the last item in the array. :::: @@ -817,14 +817,14 @@ When rendered the template outputs: ``` -#### Lambdas [search-template-lambdas] +#### Lambdas [search-template-lambdas] {{es}} has pre-built custom functions to support converting the text into a specific format. To Learn more about usage of mustache lambdas, check out the examples in [Url encode strings](#search-template-url-encode-strings), [Concatenate values](#search-template-concatenate-values), and [Convert to json](#search-template-convert-json). -### Inverted sections [search-template-inverted-section] +### Inverted sections [search-template-inverted-section] Inverted sections are useful when you want to set a value once. @@ -888,7 +888,7 @@ When rendered, template output: ``` -### Set delimiter [search-template-set-delimiter] +### Set delimiter [search-template-set-delimiter] You can change the default delimiter: double curly brackets `{{my-variable}}` to any custom delimiter in your search template. @@ -943,7 +943,7 @@ When rendered, template outputs: ``` -### Unsupported features [search-template-unsupported-features] +### Unsupported features [search-template-unsupported-features] The following mustache features are not supported in {{es}} search templates: diff --git a/solutions/search/vector/knn.md b/solutions/search/vector/knn.md index f6df5ef132..b11543a8a5 100644 --- a/solutions/search/vector/knn.md +++ b/solutions/search/vector/knn.md @@ -260,6 +260,7 @@ PUT quantized-image-index } } ``` +% TEST[continued] 1. Index your `float` vectors. @@ -347,6 +348,7 @@ POST image-index/_search "_source": false } ``` +% TEST[continued] ::::{note} The filter is applied **during** the approximate kNN search to ensure that `k` matching documents are returned. This contrasts with a post-filtering approach, where the filter is applied **after** the approximate kNN search completes. Post-filtering has the downside that it sometimes returns fewer than k results, even when there are enough matching documents. @@ -389,6 +391,7 @@ POST image-index/_search "size": 10 } ``` +% TEST[continued] This search finds the global top `k = 5` vector matches, combines them with the matches from the `match` query, and finally returns the 10 top-scoring results. The `knn` and `query` matches are combined through a disjunction, as if you took a boolean *or* between them. The top `k` vector results represent the global nearest neighbors across all index shards. @@ -440,6 +443,7 @@ Reference the deployed text embedding model or the model deployment in the `quer } (...) ``` +% NOTCONSOLE 1. The {{nlp}} task to perform. It must be `text_embedding`. 2. The ID of the text embedding model to use to generate the dense vectors from the query string. Use the same model that generated the embeddings from the input text in the index you search against. You can use the value of the `deployment_id` instead in the `model_id` argument. @@ -481,6 +485,7 @@ POST image-index/_search "size": 10 } ``` +% TEST[continued] This search finds the global top `k = 5` vector matches for `image-vector` and the global `k = 10` for the `title-vector`. These top values are then combined with the matches from the `match` query and the top-10 documents are returned. The multiple `knn` entries and the `query` matches are combined through a disjunction, as if you took a boolean *or* between them. The top `k` vector results represent the global nearest neighbors across all index shards. @@ -537,6 +542,7 @@ POST image-index/_search "_source": false } ``` +% TEST[continued] In our data set, the only document with the file type of `png` has a vector of `[42, 8, -15]`. The `l2_norm` distance between `[42, 8, -15]` and `[1, 5, -20]` is `41.412`, which is greater than the configured similarity of `36`. Meaning, this search will return no hits. @@ -578,6 +584,7 @@ PUT passage_vectors } } ``` +% TEST[continued] With the above mapping, we can index multiple passage vectors along with storing the individual passage text. @@ -588,6 +595,8 @@ POST passage_vectors/_bulk?refresh=true { "index": { "_id": "2" } } { "full_text": "number one paragraph number two paragraph", "creation_time": "2020-05-04", "paragraph": [ { "vector": [ 1.2, 4.5 ], "text": "number one paragraph", "paragraph_id": "1" }, { "vector": [ -1, 42 ], "text": "number two paragraph", "paragraph_id": "2" } ] } ``` +% TEST[continued] +% TEST[s/...//] The query will seem very similar to a typical kNN search: @@ -607,6 +616,7 @@ POST passage_vectors/_search } } ``` +% TEST[continued] Note below that even though we have 4 total vectors, we still return two documents. kNN search over nested dense_vectors will always diversify the top results over the top-level document. Meaning, `"k"` top-level documents will be returned, scored by their nearest passage vector (e.g. `"paragraph.vector"`). @@ -657,6 +667,7 @@ Note below that even though we have 4 total vectors, we still return two documen } } ``` +% TESTRESPONSE[s/"took": 4/"took" : "$body.took"/] What if you wanted to filter by some top-level document metadata? You can do this by adding `filter` to your `knn` clause. @@ -698,6 +709,7 @@ POST passage_vectors/_search } } ``` +% TEST[continued] Now we have filtered based on the top level `"creation_time"` and only one document falls within that range. @@ -772,6 +784,7 @@ POST passage_vectors/_search } } ``` +% TEST[continued] Now the result will contain the nearest found paragraph when searching. @@ -884,6 +897,7 @@ Now the result will contain the nearest found paragraph when searching. } } ``` +% TESTRESPONSE[s/"took": 4/"took" : "$body.took"/] ### Limitations for approximate kNN search [approximate-knn-limitations] @@ -937,6 +951,9 @@ POST image-index/_search "fields": [ "title", "file-type" ] } ``` +% TEST[continued] +% TEST[s/"k": 10/"k": 3/] +% TEST[s/"num_candidates": 100/"num_candidates": 3/] This example will: @@ -991,6 +1008,7 @@ POST /my-index/_search } } ``` +% TEST[skip: setup not provided] 1. The number of results to return, note its only 10 and we will oversample by 2x, gathering 20 nearest neighbors. 2. The number of results to return from the KNN search. This will do an approximate KNN search with 50 candidates per HNSW graph and use the quantized vectors, returning the 20 most similar vectors according to the quantized score. Additionally, since this is the top-level `knn` object, the global top 20 results will from all shards will be gathered before rescoring. Combining with `rescore`, this is oversampling by `2x`, meaning gathering 20 nearest neighbors according to quantized scoring and rescoring with higher fidelity float vectors. @@ -1030,6 +1048,7 @@ POST /my-index/_search } } ``` +% TEST[skip: setup not provided] 1. The number of results to return 2. The `knn` query to perform the initial search, this is executed per-shard diff --git a/troubleshoot/elasticsearch/increase-tier-capacity.md b/troubleshoot/elasticsearch/increase-tier-capacity.md index 16a0368063..f19c99189d 100644 --- a/troubleshoot/elasticsearch/increase-tier-capacity.md +++ b/troubleshoot/elasticsearch/increase-tier-capacity.md @@ -39,7 +39,6 @@ To inspect which tier an index is targeting for assignment, use the [get index s ```console GET /my-index-000001/_settings/index.routing.allocation.include._tier_preference?flat_settings ``` -% TEST[continued] % TESTRESPONSE[skip:the result is for illustrating purposes only as don’t want to change a cluster-wide setting] The response will look like this: @@ -54,7 +53,6 @@ The response will look like this: } ``` % TESTRESPONSE[skip:the result is for illustrating purposes only] -% TESTRESPONSE[skip:the result is for illustrating purposes only] 1. Represents a comma separated list of data tier node roles this index is allowed to be allocated on, the first one in the list being the one with the higher priority i.e. the tier the index is targeting. e.g. in this example the tier preference is `data_warm,data_hot` so the index is targeting the `warm` tier and more nodes with the `data_warm` role are needed in the {{es}} cluster. @@ -138,6 +136,7 @@ To inspect which tier an index is targeting for assignment, use the [get index s ```console GET /my-index-000001/_settings/index.routing.allocation.include._tier_preference?flat_settings ``` +% TEST[continued] The response will look like this: @@ -150,6 +149,7 @@ The response will look like this: } } ``` +% TESTRESPONSE[skip:the result is for illustrating purposes only] 1. Represents a comma separated list of data tier node roles this index is allowed to be allocated on, the first one in the list being the one with the higher priority i.e. the tier the index is targeting. e.g. in this example the tier preference is `data_warm,data_hot` so the index is targeting the `warm` tier and more nodes with the `data_warm` role are needed in the {{es}} cluster. diff --git a/troubleshoot/elasticsearch/remote-clusters.md b/troubleshoot/elasticsearch/remote-clusters.md index de50500995..91b655d6f7 100644 --- a/troubleshoot/elasticsearch/remote-clusters.md +++ b/troubleshoot/elasticsearch/remote-clusters.md @@ -24,7 +24,6 @@ A successful call to the cluster settings update API for adding or updating remo GET /_remote/info ``` % TEST[skip:TODO] -% TEST[skip:TODO] The API should return `"connected" : true`. When using [API key authentication](../../deploy-manage/remote-clusters/remote-clusters-api-key.md), it should also return `"cluster_credentials": "::es_redacted::"`. @@ -45,7 +44,6 @@ The API should return `"connected" : true`. When using [API key authentication]( } ``` % TEST[skip:TODO] -% TEST[skip:TODO] 1. The remote cluster has connected successfully. 2. If present, indicates the remote cluster has connected using [API key authentication](../../deploy-manage/remote-clusters/remote-clusters-api-key.md) instead of [certificate based authentication](../../deploy-manage/remote-clusters/remote-clusters-cert.md). @@ -250,6 +248,7 @@ A local cluster uses the presence of a cross-cluster API key to determine the mo ```console GET /_remote/info ``` +% TEST[skip:TODO] The API should return `"connected" : true`. When using [API key authentication](../../deploy-manage/remote-clusters/remote-clusters-api-key.md), it should also return `"cluster_credentials": "::es_redacted::"`. @@ -269,6 +268,7 @@ The API should return `"connected" : true`. When using [API key authentication]( } } ``` +% TEST[skip:TODO] 1. The remote cluster has connected successfully. 2. If present, indicates the remote cluster has connected using [API key authentication](../../deploy-manage/remote-clusters/remote-clusters-api-key.md) instead of [certificate based authentication](../../deploy-manage/remote-clusters/remote-clusters-cert.md). diff --git a/troubleshoot/elasticsearch/repeated-snapshot-failures.md b/troubleshoot/elasticsearch/repeated-snapshot-failures.md index 219a4e7602..f958699cfe 100644 --- a/troubleshoot/elasticsearch/repeated-snapshot-failures.md +++ b/troubleshoot/elasticsearch/repeated-snapshot-failures.md @@ -153,7 +153,6 @@ The response will look like this: } ``` % TESTRESPONSE[skip:the result is for illustrating purposes only] -% TESTRESPONSE[skip:the result is for illustrating purposes only] 1. The affected snapshot lifecycle policy. 2. The information about the last failure for the policy. diff --git a/troubleshoot/elasticsearch/start-ilm.md b/troubleshoot/elasticsearch/start-ilm.md index 642dbac5d7..4f4f3956ca 100644 --- a/troubleshoot/elasticsearch/start-ilm.md +++ b/troubleshoot/elasticsearch/start-ilm.md @@ -87,6 +87,7 @@ The response will look like this: "acknowledged": true } ``` +% TESTRESPONSE[skip:the result is for illustrating purposes only] Verify {{ilm}} is now running: @@ -101,6 +102,8 @@ The response will look like this: "operation_mode": "RUNNING" } ``` +% TESTRESPONSE[skip:the result is for illustrating purposes only] +:::::: :::::: ::::::: @@ -175,6 +178,7 @@ The response will look like this: "acknowledged": true } ``` +% TESTRESPONSE[skip:the result is for illustrating purposes only] Verify the {{slm}} is now running: @@ -189,6 +193,8 @@ The response will look like this: "operation_mode": "RUNNING" } ``` +% TESTRESPONSE[skip:the result is for illustrating purposes only] +:::::: :::::: ::::::: From 98db2332af9503156df2acb2f76a248a5733544a Mon Sep 17 00:00:00 2001 From: Colleen McGinnis Date: Thu, 6 Mar 2025 21:08:01 -0600 Subject: [PATCH 3/4] account for indentation --- .../full-cluster-restart-rolling-restart-procedures.md | 3 +++ .../stack-monitoring/es-legacy-collection-methods.md | 1 + deploy-manage/security/different-ca.md | 2 ++ deploy-manage/security/same-ca.md | 2 ++ .../tools/snapshot-and-restore/restore-snapshot.md | 1 + .../snapshot-and-restore/shared-file-system-repository.md | 2 ++ .../users-roles/cluster-or-deployment-auth/built-in-sm.md | 1 + .../users-roles/cluster-or-deployment-auth/jwt.md | 1 + .../users-roles/cluster-or-deployment-auth/saml.md | 1 + .../query-filter/languages/sql-getting-started.md | 2 ++ explore-analyze/scripting/scripting-field-extraction.md | 1 + explore-analyze/transforms/ecommerce-transforms.md | 1 + explore-analyze/transforms/transform-painless-examples.md | 1 + manage-data/data-store/data-streams/modify-data-stream.md | 1 + .../templates/ignore-missing-component-templates.md | 2 ++ manage-data/ingest/transform-enrich/example-parse-logs.md | 1 + manage-data/ingest/transform-enrich/ingest-pipelines.md | 1 + .../data-stream/tutorial-data-stream-retention.md | 1 + .../configure-lifecycle-policy.md | 2 ++ solutions/search/retrievers-examples.md | 2 ++ .../search/search-applications/search-application-api.md | 1 + solutions/search/vector/knn.md | 7 +++++++ troubleshoot/elasticsearch/add-tier.md | 1 + troubleshoot/elasticsearch/repeated-snapshot-failures.md | 1 + troubleshoot/elasticsearch/restore-from-snapshot.md | 6 ++++++ .../troubleshooting-shards-capacity-issues.md | 1 + troubleshoot/elasticsearch/watcher-troubleshooting.md | 1 + 27 files changed, 47 insertions(+) diff --git a/deploy-manage/maintenance/start-stop-services/full-cluster-restart-rolling-restart-procedures.md b/deploy-manage/maintenance/start-stop-services/full-cluster-restart-rolling-restart-procedures.md index 31bd0123cc..85ff5b398c 100644 --- a/deploy-manage/maintenance/start-stop-services/full-cluster-restart-rolling-restart-procedures.md +++ b/deploy-manage/maintenance/start-stop-services/full-cluster-restart-rolling-restart-procedures.md @@ -84,6 +84,7 @@ Nodes exceeding the low watermark threshold will be slow to restart. Reduce the GET _cat/nodes ``` + % TEST[continued] The `status` column returned by `_cat/health` shows the health of each node in the cluster: `red`, `yellow`, or `green`. @@ -118,6 +119,7 @@ Nodes exceeding the low watermark threshold will be slow to restart. Reduce the ```console POST _ml/set_upgrade_mode?enabled=false ``` + % TEST[continued] If you closed all {{ml}} jobs before stopping the nodes, open the jobs and start the datafeeds from {{kib}} or with the [open jobs](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-ml-open-job) and [start datafeed](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-ml-start-datafeed) APIs. @@ -208,5 +210,6 @@ Nodes exceeding the low watermark threshold will be slow to restart. Reduce the ```console POST _ml/set_upgrade_mode?enabled=false ``` + % TEST[continued] If you closed all {{ml}} jobs before stopping the nodes, open the jobs and start the datafeeds from {{kib}} or with the [open jobs](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-ml-open-job) and [start datafeed](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-ml-start-datafeed) APIs. diff --git a/deploy-manage/monitor/stack-monitoring/es-legacy-collection-methods.md b/deploy-manage/monitor/stack-monitoring/es-legacy-collection-methods.md index 546caa3e66..b1163c50f3 100644 --- a/deploy-manage/monitor/stack-monitoring/es-legacy-collection-methods.md +++ b/deploy-manage/monitor/stack-monitoring/es-legacy-collection-methods.md @@ -49,6 +49,7 @@ To learn about monitoring in general, see [Monitor a cluster](../../monitor.md). ```console GET _cluster/settings ``` + % TEST[warning:[xpack.monitoring.collection.enabled] setting was deprecated in Elasticsearch and will be removed in a future release.] ```console PUT _cluster/settings diff --git a/deploy-manage/security/different-ca.md b/deploy-manage/security/different-ca.md index 1b935d4a28..de58fdd7b3 100644 --- a/deploy-manage/security/different-ca.md +++ b/deploy-manage/security/different-ca.md @@ -145,6 +145,7 @@ If your organization has its own CA, you’ll need to [generate Certificate Sign ```console GET /_ssl/certificates ``` + % TEST[skip:creates a lot of noise] 7. If you’re only updating certificates for the transport layer (and not the HTTP layer), then complete [step 2](#start-rolling-restart-newca) through [step 6](#verify-keystore-newca) one node at a time until you’ve updated all keystores in your cluster. You can then complete the remaining steps for a [rolling restart](../maintenance/start-stop-services/full-cluster-restart-rolling-restart-procedures.md#restart-cluster-rolling). @@ -275,6 +276,7 @@ This process is different for each client, so refer to your client’s documenta ```console GET /_ssl/certificates ``` + % TEST[skip:creates a lot of noise] 11. One node at a time, complete [step 5](#start-rolling-restart-http-newca) through [step 10](#verify-keystore-http-newca) until you’ve updated all keystores in your cluster. 12. Complete the remaining steps for a [rolling restart](../maintenance/start-stop-services/full-cluster-restart-rolling-restart-procedures.md#restart-cluster-rolling), beginning with the step to **Reenable shard allocation**. diff --git a/deploy-manage/security/same-ca.md b/deploy-manage/security/same-ca.md index dc1379320a..1510c7a848 100644 --- a/deploy-manage/security/same-ca.md +++ b/deploy-manage/security/same-ca.md @@ -98,6 +98,7 @@ The following examples use PKCS#12 files, but the same steps apply to JKS keysto ```console GET /_ssl/certificates ``` + % TEST[skip:creates a lot of noise] 9. If you’re only updating certificates for the transport layer (and not the HTTP layer), then complete [step 4](#start-rolling-restart) through [step 8](#verify-keystore) one node at a time until you’ve updated all keystores in your cluster. You can then complete the remaining steps for a [rolling restart](../maintenance/start-stop-services/full-cluster-restart-rolling-restart-procedures.md#restart-cluster-rolling). @@ -205,6 +206,7 @@ If your organization has its own CA, you’ll need to [generate Certificate Sign ```console GET /_ssl/certificates ``` + % TEST[skip:creates a lot of noise] 11. One node at a time, complete [step 5](#start-rolling-restart-http) through [step 10](#verify-keystore-http) until you’ve updated all keystores in your cluster. 12. Complete the remaining steps for a [rolling restart](../maintenance/start-stop-services/full-cluster-restart-rolling-restart-procedures.md#restart-cluster-rolling), beginning with the step to **Reenable shard allocation**. diff --git a/deploy-manage/tools/snapshot-and-restore/restore-snapshot.md b/deploy-manage/tools/snapshot-and-restore/restore-snapshot.md index 17ac58d95f..f04ee2db7f 100644 --- a/deploy-manage/tools/snapshot-and-restore/restore-snapshot.md +++ b/deploy-manage/tools/snapshot-and-restore/restore-snapshot.md @@ -465,6 +465,7 @@ DELETE my-index # Delete a data stream DELETE _data_stream/logs-my_app-default ``` +% TEST[setup:setup-snapshots] ## Restore to a different cluster [restore-different-cluster] diff --git a/deploy-manage/tools/snapshot-and-restore/shared-file-system-repository.md b/deploy-manage/tools/snapshot-and-restore/shared-file-system-repository.md index 5cf7bf1b42..c001d840df 100644 --- a/deploy-manage/tools/snapshot-and-restore/shared-file-system-repository.md +++ b/deploy-manage/tools/snapshot-and-restore/shared-file-system-repository.md @@ -139,6 +139,8 @@ PUT _snapshot/my_fs_backup } } ``` +% TEST[skip:no access to path] +:::::: :::::: ::::::: diff --git a/deploy-manage/users-roles/cluster-or-deployment-auth/built-in-sm.md b/deploy-manage/users-roles/cluster-or-deployment-auth/built-in-sm.md index a96b7646c9..6e1f5adaa2 100644 --- a/deploy-manage/users-roles/cluster-or-deployment-auth/built-in-sm.md +++ b/deploy-manage/users-roles/cluster-or-deployment-auth/built-in-sm.md @@ -58,6 +58,7 @@ POST /_security/user/user1/_password "password" : "new-test-password" } ``` +% TEST[continued] For more information and examples, see [Users](https://www.elastic.co/docs/api/doc/elasticsearch/group/endpoint-security). diff --git a/deploy-manage/users-roles/cluster-or-deployment-auth/jwt.md b/deploy-manage/users-roles/cluster-or-deployment-auth/jwt.md index 21806d84bb..e89c88bbdf 100644 --- a/deploy-manage/users-roles/cluster-or-deployment-auth/jwt.md +++ b/deploy-manage/users-roles/cluster-or-deployment-auth/jwt.md @@ -479,6 +479,7 @@ Audiences: aud8 Algorithms: HS256 HMAC UTF-8: hmac-oidc-key-string-for-hs256-algorithm ``` +% NOTCONSOLE ### JWT realm settings [hmac-oidc-example-jwt-realm] diff --git a/deploy-manage/users-roles/cluster-or-deployment-auth/saml.md b/deploy-manage/users-roles/cluster-or-deployment-auth/saml.md index e1314621bd..fdffa22850 100644 --- a/deploy-manage/users-roles/cluster-or-deployment-auth/saml.md +++ b/deploy-manage/users-roles/cluster-or-deployment-auth/saml.md @@ -811,6 +811,7 @@ POST /_security/saml/authenticate "ids" : [] } ``` +% TEST[skip:handled in IT] ### Handling the logout flow [saml-no-kibana-slo] diff --git a/explore-analyze/query-filter/languages/sql-getting-started.md b/explore-analyze/query-filter/languages/sql-getting-started.md index cd4d1e946c..d2b5e21336 100644 --- a/explore-analyze/query-filter/languages/sql-getting-started.md +++ b/explore-analyze/query-filter/languages/sql-getting-started.md @@ -57,4 +57,6 @@ sql> SELECT * FROM library WHERE release_date < '2000-01-01'; Dan Simmons |Hyperion |482 |1989-05-26T00:00:00.000Z Frank Herbert |Dune |604 |1965-06-01T00:00:00.000Z ``` +% TESTRESPONSE[s/|/\|/ s/+/\+/] +% TESTRESPONSE[non_json] diff --git a/explore-analyze/scripting/scripting-field-extraction.md b/explore-analyze/scripting/scripting-field-extraction.md index 6e329e3cfe..0604a8cb15 100644 --- a/explore-analyze/scripting/scripting-field-extraction.md +++ b/explore-analyze/scripting/scripting-field-extraction.md @@ -240,6 +240,7 @@ Looking at the data again, there’s a timestamp, some other data that you’re ```txt [2021-04-27T16:16:34.699+0000][82460][gc,heap,exit] class space used 266K, capacity 384K, committed 384K, reserved 1048576K ``` +% NOTCONSOLE You can assign variables to each part of the data in the `gc` field, and then return only the parts that you want. Anything in curly braces `{}` is considered a variable. For example, the variables `[%{@timestamp}][%{{code}}][%{{desc}}]` will match the first three chunks of data, all of which are in square brackets `[]`. diff --git a/explore-analyze/transforms/ecommerce-transforms.md b/explore-analyze/transforms/ecommerce-transforms.md index fb82e8af3a..4061c36092 100644 --- a/explore-analyze/transforms/ecommerce-transforms.md +++ b/explore-analyze/transforms/ecommerce-transforms.md @@ -307,6 +307,7 @@ mapped_pages: ```console POST _transform/ecommerce-customer-transform/_start ``` + % TEST[skip:setup kibana sample data] :::: diff --git a/explore-analyze/transforms/transform-painless-examples.md b/explore-analyze/transforms/transform-painless-examples.md index 5cb458981e..ab570f05a3 100644 --- a/explore-analyze/transforms/transform-painless-examples.md +++ b/explore-analyze/transforms/transform-painless-examples.md @@ -173,6 +173,7 @@ You can also use the power of [stored scripts](https://www.elastic.co/docs/api/d "id":"last-value-reduce" } ``` + % NOTCONSOLE 1. The parameter `field_with_last_value` can be set any field that you want the latest value for. diff --git a/manage-data/data-store/data-streams/modify-data-stream.md b/manage-data/data-store/data-streams/modify-data-stream.md index 7adc165b71..65425e898c 100644 --- a/manage-data/data-store/data-streams/modify-data-stream.md +++ b/manage-data/data-store/data-streams/modify-data-stream.md @@ -340,6 +340,7 @@ Follow these steps: ```console PUT /_data_stream/new-data-stream ``` + % TEST[s/new-data-stream/new-data-stream-two/] 4. If you do not want to mix new and old data in your new data stream, pause the indexing of new documents. While mixing old and new data is safe, it could interfere with data retention. See [Mixing new and old data in a data stream](../data-streams/modify-data-stream.md#data-stream-mix-new-old-data). 5. If you use {{ilm-init}} to [automate rollover](../../lifecycle/index-lifecycle-management/tutorial-automate-rollover.md), reduce the {{ilm-init}} poll interval. This ensures the current write index doesn’t grow too large while waiting for the rollover check. By default, {{ilm-init}} checks rollover conditions every 10 minutes. diff --git a/manage-data/data-store/templates/ignore-missing-component-templates.md b/manage-data/data-store/templates/ignore-missing-component-templates.md index 1940fba0c6..2cde3a2dac 100644 --- a/manage-data/data-store/templates/ignore-missing-component-templates.md +++ b/manage-data/data-store/templates/ignore-missing-component-templates.md @@ -102,4 +102,6 @@ This will not have an immediate effect on the data stream. The mapping `host.ip` ```console POST logs-foo-bar/_rollover ``` +% TEST[continued] +% TEST[teardown:data_stream_cleanup] diff --git a/manage-data/ingest/transform-enrich/example-parse-logs.md b/manage-data/ingest/transform-enrich/example-parse-logs.md index b465eb9dd7..d61da8754c 100644 --- a/manage-data/ingest/transform-enrich/example-parse-logs.md +++ b/manage-data/ingest/transform-enrich/example-parse-logs.md @@ -141,6 +141,7 @@ These logs contain a timestamp, IP address, and user agent. You want to give the ```console GET my-data-stream/_search?filter_path=hits.hits._source ``` + % TEST[continued] The API returns: diff --git a/manage-data/ingest/transform-enrich/ingest-pipelines.md b/manage-data/ingest/transform-enrich/ingest-pipelines.md index b66bfdb6ee..31c6eed7cb 100644 --- a/manage-data/ingest/transform-enrich/ingest-pipelines.md +++ b/manage-data/ingest/transform-enrich/ingest-pipelines.md @@ -321,6 +321,7 @@ $$$pipeline-custom-logs-index-template$$$ ```console POST logs-my_app-default/_rollover/ ``` + % TEST[continued] $$$pipeline-custom-logs-configuration$$$ diff --git a/manage-data/lifecycle/data-stream/tutorial-data-stream-retention.md b/manage-data/lifecycle/data-stream/tutorial-data-stream-retention.md index e5fb3a21db..fe9dad80a5 100644 --- a/manage-data/lifecycle/data-stream/tutorial-data-stream-retention.md +++ b/manage-data/lifecycle/data-stream/tutorial-data-stream-retention.md @@ -138,6 +138,7 @@ Considering our example, if we retrieve the lifecycle of `my-data-stream`: ```console GET _data_stream/my-data-stream/_lifecycle ``` +% TEST[continued] We see that it will remain the same with what the user configured: diff --git a/manage-data/lifecycle/index-lifecycle-management/configure-lifecycle-policy.md b/manage-data/lifecycle/index-lifecycle-management/configure-lifecycle-policy.md index 8c74f4c064..48025b82b9 100644 --- a/manage-data/lifecycle/index-lifecycle-management/configure-lifecycle-policy.md +++ b/manage-data/lifecycle/index-lifecycle-management/configure-lifecycle-policy.md @@ -213,12 +213,14 @@ To switch an index’s lifecycle policy, follow these steps: ```console GET logs-my_app-default ``` + % TEST[continued] You can then change the index as needed. For example, you can re-open any closed indices using the [open index API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-open). ```console POST logs-my_app-default/_open ``` + % TEST[continued] 3. Assign a new policy using the [update settings API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-put-settings). Target a data stream or alias to assign a policy to all its indices. diff --git a/solutions/search/retrievers-examples.md b/solutions/search/retrievers-examples.md index 5bdf1def0c..35e373ee78 100644 --- a/solutions/search/retrievers-examples.md +++ b/solutions/search/retrievers-examples.md @@ -1362,6 +1362,7 @@ GET retrievers_example/_search "_source": false } ``` +% TEST[skip: no_access_to_ml] ## Example: RRF with semantic reranker [retrievers-examples-rrf-ranking-on-text-similarity-reranker-results] @@ -1410,6 +1411,7 @@ GET /retrievers_example/_search "_source": false } ``` +% TEST[skip: no_access_to_ml] ## Example: Chaining multiple semantic rerankers [retrievers-examples-chaining-text-similarity-reranker-retrievers] diff --git a/solutions/search/search-applications/search-application-api.md b/solutions/search/search-applications/search-application-api.md index 75694eff22..dd6f83ab7e 100644 --- a/solutions/search/search-applications/search-application-api.md +++ b/solutions/search/search-applications/search-application-api.md @@ -545,6 +545,7 @@ Finally, a parameterless search using this template would fall back to a default ```console POST _application/search_application/my_search_application/_search ``` +% TEST[continued] ### ELSER search [search-application-api-elser-template] diff --git a/solutions/search/vector/knn.md b/solutions/search/vector/knn.md index b11543a8a5..a20d2a50fe 100644 --- a/solutions/search/vector/knn.md +++ b/solutions/search/vector/knn.md @@ -226,6 +226,9 @@ POST byte-image-index/_search "fields": [ "title" ] } ``` +% TEST[continued] +% TEST[s/"k": 10/"k": 3/] +% TEST[s/"num_candidates": 100/"num_candidates": 3/] ### Byte quantized kNN search [knn-search-quantized-example] @@ -322,6 +325,9 @@ POST quantized-image-index/_search } } ``` +% TEST[continued] +% TEST[s/"k": 15/"k": 3/] +% TEST[s/"num_candidates": 100/"num_candidates": 3/] ### Filtered kNN search [knn-search-filter-example] @@ -747,6 +753,7 @@ Now we have filtered based on the top level `"creation_time"` and only one docum } } ``` +% TESTRESPONSE[s/"took": 4/"took" : "$body.took"/] ### Nested kNN Search with Inner hits [nested-knn-search-inner-hits] diff --git a/troubleshoot/elasticsearch/add-tier.md b/troubleshoot/elasticsearch/add-tier.md index 851cf6a7f5..bbfcd7a66e 100644 --- a/troubleshoot/elasticsearch/add-tier.md +++ b/troubleshoot/elasticsearch/add-tier.md @@ -40,6 +40,7 @@ In order to get the shards assigned we need enable a new tier in the deployment. ```console GET /my-index-000001/_settings/index.routing.allocation.include._tier_preference?flat_settings ``` + % TEST[continued] The response will look like this: diff --git a/troubleshoot/elasticsearch/repeated-snapshot-failures.md b/troubleshoot/elasticsearch/repeated-snapshot-failures.md index f958699cfe..c363a6f27e 100644 --- a/troubleshoot/elasticsearch/repeated-snapshot-failures.md +++ b/troubleshoot/elasticsearch/repeated-snapshot-failures.md @@ -38,6 +38,7 @@ In order to check the status of failing {{slm}} policies we need to go to Kibana ```console GET _slm/policy/ ``` + % TEST[skip:These policies do not exist] The response will look like this: diff --git a/troubleshoot/elasticsearch/restore-from-snapshot.md b/troubleshoot/elasticsearch/restore-from-snapshot.md index 170f67afc2..1595d6201c 100644 --- a/troubleshoot/elasticsearch/restore-from-snapshot.md +++ b/troubleshoot/elasticsearch/restore-from-snapshot.md @@ -38,6 +38,7 @@ In order to restore the indices and data streams that are missing data: ```console GET _cat/indices?v&health=red&h=index,status,health ``` + % TEST[skip:illustration purposes only] The response will look like this: @@ -54,6 +55,7 @@ In order to restore the indices and data streams that are missing data: ```console GET _snapshot/my_repository/*?verbose=false ``` + % TEST[skip:illustration purposes only] The response will look like this: @@ -149,6 +151,7 @@ In order to restore the indices and data streams that are missing data: ```console POST my-data-stream/_rollover ``` + % TEST[skip:illustration purposes only] 8. Now that the data stream preparation is done, we will close the target indices by using the [close indices API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-close). @@ -161,6 +164,7 @@ In order to restore the indices and data streams that are missing data: ```console GET _cat/indices?v&health=red&h=index,status,health ``` + % TEST[skip:illustration purposes only] The response will look like this: @@ -242,6 +246,7 @@ In order to restore the indices that are missing shards: ```console GET _snapshot/my_repository/*?verbose=false ``` + % TEST[skip:illustration purposes only] The response will look like this: @@ -337,6 +342,7 @@ In order to restore the indices that are missing shards: ```console POST my-data-stream/_rollover ``` + % TEST[skip:illustration purposes only] 5. Now that the data stream preparation is done, we will close the target indices by using the [close indices API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-close). diff --git a/troubleshoot/elasticsearch/troubleshooting-shards-capacity-issues.md b/troubleshoot/elasticsearch/troubleshooting-shards-capacity-issues.md index 961e02a3f0..b6b861fd2b 100644 --- a/troubleshoot/elasticsearch/troubleshooting-shards-capacity-issues.md +++ b/troubleshoot/elasticsearch/troubleshooting-shards-capacity-issues.md @@ -41,6 +41,7 @@ If you’re confident your changes won’t destabilize the cluster, you can temp ```console GET _health_report/shards_capacity ``` + % TESTRESPONSE[skip:the result is for illustrating purposes only] The response will look like this: diff --git a/troubleshoot/elasticsearch/watcher-troubleshooting.md b/troubleshoot/elasticsearch/watcher-troubleshooting.md index 0ae4c06e3c..607c537e3a 100644 --- a/troubleshoot/elasticsearch/watcher-troubleshooting.md +++ b/troubleshoot/elasticsearch/watcher-troubleshooting.md @@ -26,6 +26,7 @@ If the index mappings are missing, follow these steps to restore the correct map ```console DELETE .watches ``` + % TEST[skip:index deletion and deprecation warning] 5. Disable direct access to the `.watches` index: From d1854a1b1314eb83f5ad53bb7bd8360765789675 Mon Sep 17 00:00:00 2001 From: George Wallace Date: Thu, 6 Mar 2025 21:53:20 -0700 Subject: [PATCH 4/4] fixing issues --- ...asticsearch-from-archive-on-linux-macos.md | 6 +- ...stall-elasticsearch-with-debian-package.md | 3 +- .../install-elasticsearch-with-rpm.md | 3 +- ...stall-elasticsearch-with-zip-on-windows.md | 3 +- .../remote-clusters-api-key.md | 14 ++--- .../remote-clusters/remote-clusters-cert.md | 12 ++-- .../snapshot-and-restore/create-snapshots.md | 3 +- .../snapshot-and-restore/restore-snapshot.md | 7 +-- .../shared-file-system-repository.md | 2 - .../service-accounts.md | 3 +- .../alerts-cases/watcher/actions-email.md | 5 +- .../alerts-cases/watcher/actions-index.md | 4 +- .../alerts-cases/watcher/actions-jira.md | 2 +- .../alerts-cases/watcher/actions-logging.md | 2 +- .../alerts-cases/watcher/actions-pagerduty.md | 2 +- .../alerts-cases/watcher/actions-slack.md | 5 +- .../alerts-cases/watcher/actions-webhook.md | 8 +-- .../watcher/condition-array-compare.md | 2 +- .../alerts-cases/watcher/condition-compare.md | 2 +- .../alerts-cases/watcher/how-watcher-works.md | 3 +- .../alerts-cases/watcher/input-chain.md | 2 +- .../alerts-cases/watcher/input-http.md | 2 +- .../alerts-cases/watcher/transform-chain.md | 2 +- .../alerts-cases/watcher/transform-script.md | 2 +- .../alerts-cases/watcher/transform.md | 2 +- .../elasticsearch-inference-integration.md | 15 ++--- .../huggingface-inference-integration.md | 3 +- .../mistral-inference-integration.md | 3 +- explore-analyze/query-filter/aggregations.md | 14 ++--- ...-data-with-aggregations-using-query-dsl.md | 30 ++++----- .../languages/esql-cross-clusters.md | 8 +-- .../languages/esql-task-management.md | 3 +- .../example-detect-threats-with-eql.md | 3 +- .../transforms/transform-examples.md | 25 ++++---- .../transforms/transform-painless-examples.md | 20 +++--- .../data-streams/logs-data-stream.md | 3 +- .../data-streams/run-downsampling-manually.md | 5 +- .../data-streams/use-data-stream.md | 28 ++++----- .../data-store/mapping/dynamic-templates.md | 3 +- ...ed-data-stream-to-data-stream-lifecycle.md | 48 +++++++------- .../tutorial-update-existing-data-stream.md | 8 +-- .../configure-lifecycle-policy.md | 3 +- .../tutorial-automate-rollover.md | 9 +-- solutions/search/cross-cluster-search.md | 62 ++++++++----------- .../static-scoring-signals.md | 3 +- solutions/search/hybrid-semantic-text.md | 3 +- .../querydsl-full-text-filter-tutorial.md | 44 +++++-------- .../ranking/learning-to-rank-search-usage.md | 3 +- .../search-application-security.md | 3 +- .../semantic-search-elser-ingest-pipelines.md | 9 +-- .../semantic-search-semantic-text.md | 9 +-- solutions/search/vector/bring-own-vectors.md | 9 +-- .../dense-versus-sparse-ingest-pipelines.md | 8 +-- solutions/search/vector/knn.md | 11 ++-- troubleshoot/elasticsearch/add-tier.md | 3 +- .../elasticsearch/increase-shard-limit.md | 5 +- .../elasticsearch/increase-tier-capacity.md | 6 +- .../index-lifecycle-management-errors.md | 3 +- troubleshoot/elasticsearch/remote-clusters.md | 6 +- .../repeated-snapshot-failures.md | 3 +- troubleshoot/elasticsearch/start-ilm.md | 4 +- .../troubleshooting-shards-capacity-issues.md | 6 +- 62 files changed, 224 insertions(+), 308 deletions(-) diff --git a/deploy-manage/deploy/self-managed/install-elasticsearch-from-archive-on-linux-macos.md b/deploy-manage/deploy/self-managed/install-elasticsearch-from-archive-on-linux-macos.md index c340fa9aad..9029764d82 100644 --- a/deploy-manage/deploy/self-managed/install-elasticsearch-from-archive-on-linux-macos.md +++ b/deploy-manage/deploy/self-managed/install-elasticsearch-from-archive-on-linux-macos.md @@ -69,11 +69,10 @@ curl https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-9.0.0-be tar -xzf elasticsearch-9.0.0-beta1-darwin-x86_64.tar.gz cd elasticsearch-9.0.0-beta1/ <2> ``` -% NOTCONSOLE 1. Compares the SHA of the downloaded `.tar.gz` archive and the published checksum, which should output `elasticsearch-{{version}}-darwin-x86_64.tar.gz: OK`. 2. This directory is known as `$ES_HOME`. - +% NOTCONSOLE ## Enable automatic creation of system indices [targz-enable-indices] @@ -165,11 +164,10 @@ You can test that your {{es}} node is running by sending an HTTPS request to por ```sh curl --cacert $ES_HOME/config/certs/http_ca.crt -u elastic:$ELASTIC_PASSWORD https://localhost:9200 <1> ``` -% NOTCONSOLE 1. Ensure that you use `https` in your call, or the request will fail.`--cacert` : Path to the generated `http_ca.crt` certificate for the HTTP layer. - +% NOTCONSOLE The call returns a response like this: diff --git a/deploy-manage/deploy/self-managed/install-elasticsearch-with-debian-package.md b/deploy-manage/deploy/self-managed/install-elasticsearch-with-debian-package.md index 58d6834d36..6f59238e75 100644 --- a/deploy-manage/deploy/self-managed/install-elasticsearch-with-debian-package.md +++ b/deploy-manage/deploy/self-managed/install-elasticsearch-with-debian-package.md @@ -211,11 +211,10 @@ You can test that your {{es}} node is running by sending an HTTPS request to por ```sh curl --cacert /etc/elasticsearch/certs/http_ca.crt -u elastic:$ELASTIC_PASSWORD https://localhost:9200 <1> ``` -% NOTCONSOLE 1. Ensure that you use `https` in your call, or the request will fail.`--cacert` : Path to the generated `http_ca.crt` certificate for the HTTP layer. - +% NOTCONSOLE The call returns a response like this: diff --git a/deploy-manage/deploy/self-managed/install-elasticsearch-with-rpm.md b/deploy-manage/deploy/self-managed/install-elasticsearch-with-rpm.md index a3288c8bf6..5de8b21ea5 100644 --- a/deploy-manage/deploy/self-managed/install-elasticsearch-with-rpm.md +++ b/deploy-manage/deploy/self-managed/install-elasticsearch-with-rpm.md @@ -215,11 +215,10 @@ You can test that your {{es}} node is running by sending an HTTPS request to por ```sh curl --cacert /etc/elasticsearch/certs/http_ca.crt -u elastic:$ELASTIC_PASSWORD https://localhost:9200 <1> ``` -% NOTCONSOLE 1. Ensure that you use `https` in your call, or the request will fail.`--cacert` : Path to the generated `http_ca.crt` certificate for the HTTP layer. - +% NOTCONSOLE The call returns a response like this: diff --git a/deploy-manage/deploy/self-managed/install-elasticsearch-with-zip-on-windows.md b/deploy-manage/deploy/self-managed/install-elasticsearch-with-zip-on-windows.md index b905ff7c78..906e3ab42e 100644 --- a/deploy-manage/deploy/self-managed/install-elasticsearch-with-zip-on-windows.md +++ b/deploy-manage/deploy/self-managed/install-elasticsearch-with-zip-on-windows.md @@ -142,11 +142,10 @@ You can test that your {{es}} node is running by sending an HTTPS request to por ```sh curl --cacert %ES_HOME%\config\certs\http_ca.crt -u elastic:$ELASTIC_PASSWORD https://localhost:9200 <1> ``` -% NOTCONSOLE 1. Ensure that you use `https` in your call, or the request will fail.`--cacert` : Path to the generated `http_ca.crt` certificate for the HTTP layer. - +% NOTCONSOLE The call returns a response like this: diff --git a/deploy-manage/remote-clusters/remote-clusters-api-key.md b/deploy-manage/remote-clusters/remote-clusters-api-key.md index 6dd62556a8..ce1713ce8a 100644 --- a/deploy-manage/remote-clusters/remote-clusters-api-key.md +++ b/deploy-manage/remote-clusters/remote-clusters-api-key.md @@ -167,12 +167,11 @@ PUT /_cluster/settings } } ``` -% TEST[setup:host] -% TEST[s/127.0.0.1:{{remote-interface-default-port}}/${transport_host}/] 1. The cluster alias of this remote cluster is `cluster_one`. 2. Specifies the hostname and remote cluster port of a seed node in the remote cluster. - +% TEST[setup:host] +% TEST[s/127.0.0.1:{{remote-interface-default-port}}/${transport_host}/] You can use the [remote cluster info API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-cluster-remote-info) to verify that the local cluster is successfully connected to the remote cluster: @@ -199,15 +198,14 @@ The API response indicates that the local cluster is connected to the remote clu } } ``` -% TESTRESPONSE[s/127.0.0.1:{{remote-interface-default-port}}/$body.cluster_one.seeds.0/] -% TESTRESPONSE[s/ifeval::(.|\n)*endif::[]//] -% TEST[s/"connected" : true/"connected" : $body.cluster_one.connected/] -% TEST[s/"num_nodes_connected" : 1/"num_nodes_connected" : $body.cluster_one.num_nodes_connected/] 1. The number of nodes in the remote cluster the local cluster is connected to. 2. Indicates whether to skip the remote cluster if searched through {{ccs}} but no nodes are available. 3. If present, indicates the remote cluster has connected using API key authentication. - +% TESTRESPONSE[s/127.0.0.1:{{remote-interface-default-port}}/$body.cluster_one.seeds.0/] +% TESTRESPONSE[s/ifeval::(.|\n)*endif::[]//] +% TEST[s/"connected" : true/"connected" : $body.cluster_one.connected/] +% TEST[s/"num_nodes_connected" : 1/"num_nodes_connected" : $body.cluster_one.num_nodes_connected/] ### Dynamically configure remote clusters [_dynamically_configure_remote_clusters] diff --git a/deploy-manage/remote-clusters/remote-clusters-cert.md b/deploy-manage/remote-clusters/remote-clusters-cert.md index 738433afe6..69474c6122 100644 --- a/deploy-manage/remote-clusters/remote-clusters-cert.md +++ b/deploy-manage/remote-clusters/remote-clusters-cert.md @@ -85,12 +85,11 @@ PUT /_cluster/settings } } ``` -% TEST[setup:host] -% TEST[s/127.0.0.1:{{remote-interface-default-port}}/${transport_host}/] 1. The cluster alias of this remote cluster is `cluster_one`. 2. Specifies the hostname and transport port of a seed node in the remote cluster. - +% TEST[setup:host] +% TEST[s/127.0.0.1:{{remote-interface-default-port}}/${transport_host}/] You can use the [remote cluster info API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-cluster-remote-info) to verify that the local cluster is successfully connected to the remote cluster: @@ -116,15 +115,14 @@ The API response indicates that the local cluster is connected to the remote clu } } ``` + +1. The number of nodes in the remote cluster the local cluster is connected to. +2. Indicates whether to skip the remote cluster if searched through {{ccs}} but no nodes are available. % TESTRESPONSE[s/127.0.0.1:{{remote-interface-default-port}}/$body.cluster_one.seeds.0/] % TESTRESPONSE[s/ifeval::(.|\n)*endif::[]//] % TEST[s/"connected" : true/"connected" : $body.cluster_one.connected/] % TEST[s/"num_nodes_connected" : 1/"num_nodes_connected" : $body.cluster_one.num_nodes_connected/] -1. The number of nodes in the remote cluster the local cluster is connected to. -2. Indicates whether to skip the remote cluster if searched through {{ccs}} but no nodes are available. - - ### Dynamically configure remote clusters [_dynamically_configure_remote_clusters_2] Use the [cluster update settings API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-cluster-put-settings) to dynamically configure remote settings on every node in the cluster. The following request adds three remote clusters: `cluster_one`, `cluster_two`, and `cluster_three`. diff --git a/deploy-manage/tools/snapshot-and-restore/create-snapshots.md b/deploy-manage/tools/snapshot-and-restore/create-snapshots.md index eb208ad15c..3c07015f55 100644 --- a/deploy-manage/tools/snapshot-and-restore/create-snapshots.md +++ b/deploy-manage/tools/snapshot-and-restore/create-snapshots.md @@ -353,11 +353,10 @@ PUT _slm/policy/nightly-cluster-state-snapshots } } ``` -% TEST[s/my_secure_repository/my_repository/] 1. Includes the cluster state. This also includes all feature states by default. 2. Excludes regular data streams and indices. - +% TEST[s/my_secure_repository/my_repository/] If you take dedicated snapshots of the cluster state, you’ll need to exclude the cluster state from your other snapshots. For example: diff --git a/deploy-manage/tools/snapshot-and-restore/restore-snapshot.md b/deploy-manage/tools/snapshot-and-restore/restore-snapshot.md index f04ee2db7f..66d6b2fc40 100644 --- a/deploy-manage/tools/snapshot-and-restore/restore-snapshot.md +++ b/deploy-manage/tools/snapshot-and-restore/restore-snapshot.md @@ -198,15 +198,14 @@ POST _snapshot/my_repository/my_snapshot_2099.05.06/_restore "indices": "-*" <2> } ``` + +1. Exclude the cluster state from the restore operation. +2. Exclude the other indices and data streams in the snapshot from the restore operation. % TEST[setup:setup-snapshots] % TEST[s/^/DELETE my-index\nDELETE _data_stream/logs-my_app-default\n/] % TEST[s/_restore/_restore?wait_for_completion=true/] % TEST[s/"feature_states": [ "geoip" ],//] -1. Exclude the cluster state from the restore operation. -2. Exclude the other indices and data streams in the snapshot from the restore operation. - - ## Restore an entire cluster [restore-entire-cluster] diff --git a/deploy-manage/tools/snapshot-and-restore/shared-file-system-repository.md b/deploy-manage/tools/snapshot-and-restore/shared-file-system-repository.md index c001d840df..9ca8dca892 100644 --- a/deploy-manage/tools/snapshot-and-restore/shared-file-system-repository.md +++ b/deploy-manage/tools/snapshot-and-restore/shared-file-system-repository.md @@ -78,7 +78,6 @@ PUT _snapshot/my_fs_backup ``` % TEST[skip:no access to path] :::::: -:::::: ::::::{tab-item} Windows Windows installations support both DOS and Microsoft UNC paths. Escape any backslashes in the paths. For UNC paths, provide the server and share name as a prefix. @@ -141,7 +140,6 @@ PUT _snapshot/my_fs_backup ``` % TEST[skip:no access to path] :::::: -:::::: ::::::: ## Repository settings [filesystem-repository-settings] diff --git a/deploy-manage/users-roles/cluster-or-deployment-auth/service-accounts.md b/deploy-manage/users-roles/cluster-or-deployment-auth/service-accounts.md index 285e4dc8a1..16054728f5 100644 --- a/deploy-manage/users-roles/cluster-or-deployment-auth/service-accounts.md +++ b/deploy-manage/users-roles/cluster-or-deployment-auth/service-accounts.md @@ -103,9 +103,8 @@ A successful authentication response includes a `token` field, which contains a "authentication_type": "token" } ``` -% NOTCONSOLE 1. Name of the service account token. 2. Type of service account token. The value always begins with `_service_account_` and is followed by a string that indicates the service token backend in use (can be either `file` or `index`). - +% NOTCONSOLE diff --git a/explore-analyze/alerts-cases/watcher/actions-email.md b/explore-analyze/alerts-cases/watcher/actions-email.md index f11ae4c623..8cbcaea1c2 100644 --- a/explore-analyze/alerts-cases/watcher/actions-email.md +++ b/explore-analyze/alerts-cases/watcher/actions-email.md @@ -32,13 +32,13 @@ For example, the following email action uses a template to include data from the } } ``` -% NOTCONSOLE 1. The id of the action. 2. The action type is set to `email`. 3. One or more addresses to send the email to. Must be specified in the action definition or in the email account configuration. 4. The subject of the email can contain static text and Mustache [templates](how-watcher-works.md#templates). 5. The body of the email can contain static text and Mustache [templates](how-watcher-works.md#templates). Must be specified in the action definition or in the email account configuration. +% NOTCONSOLE ## Configuring email attachments [configuring-email-attachments] @@ -75,13 +75,12 @@ To configure attachments, specify a name for the attached file and the type of a } } ``` -% NOTCONSOLE 1. The ID of the attachment, which is used as the file name in the email attachment. 2. The type of the attachment and its specific configuration. 3. The URL from which to retrieve the attachment. 4. Data attachments default to JSON if you don’t specify the format. - +% NOTCONSOLE | Name | Description | | --- | --- | diff --git a/explore-analyze/alerts-cases/watcher/actions-index.md b/explore-analyze/alerts-cases/watcher/actions-index.md index b77ac118b0..8fc91d1606 100644 --- a/explore-analyze/alerts-cases/watcher/actions-index.md +++ b/explore-analyze/alerts-cases/watcher/actions-index.md @@ -27,13 +27,13 @@ The following snippet shows a simple `index` action definition: } } ``` -% NOTCONSOLE 1. The id of the action 2. An optional [condition](condition.md) to restrict action execution 3. An optional [transform](transform.md) to transform the payload and prepare the data that should be indexed 4. The index, alias, or data stream to which the data will be written 5. An optional `_id` for the document +% NOTCONSOLE ## Index action attributes [index-action-attributes] @@ -79,10 +79,10 @@ The following snippet shows a multi-document `index` action definition: } } ``` -% NOTCONSOLE 1. The document’s index 2. An optional `_id` for the document 3. A new `severity` field derived from the original document 4. The payload `_doc` field which is an array of documents 5. Since the `_index` was informed per document this should be empty +% NOTCONSOLE \ No newline at end of file diff --git a/explore-analyze/alerts-cases/watcher/actions-jira.md b/explore-analyze/alerts-cases/watcher/actions-jira.md index 96fe39f386..194da12355 100644 --- a/explore-analyze/alerts-cases/watcher/actions-jira.md +++ b/explore-analyze/alerts-cases/watcher/actions-jira.md @@ -42,7 +42,6 @@ The following snippet shows a simple jira action definition: } } ``` -% NOTCONSOLE 1. The name of a Jira account configured in `elasticsearch.yml`. 2. The key of the Jira project in which the issue will be created. @@ -51,6 +50,7 @@ The following snippet shows a simple jira action definition: 5. The description of the Jira issue. 6. The labels to apply to the Jira issue. 7. The priority of the Jira issue. +% NOTCONSOLE ## Jira action attributes [jira-action-attributes] diff --git a/explore-analyze/alerts-cases/watcher/actions-logging.md b/explore-analyze/alerts-cases/watcher/actions-logging.md index 56dff578f9..2e391807c8 100644 --- a/explore-analyze/alerts-cases/watcher/actions-logging.md +++ b/explore-analyze/alerts-cases/watcher/actions-logging.md @@ -29,11 +29,11 @@ The following snippet shows a simple logging action definition: } } ``` -% NOTCONSOLE 1. The id of the action. 2. An optional [transform](transform.md) to transform the payload before executing the `logging` action. 3. The text to be logged. +% NOTCONSOLE ## Logging action attributes [logging-action-attributes] diff --git a/explore-analyze/alerts-cases/watcher/actions-pagerduty.md b/explore-analyze/alerts-cases/watcher/actions-pagerduty.md index fc78727205..c38424bf7c 100644 --- a/explore-analyze/alerts-cases/watcher/actions-pagerduty.md +++ b/explore-analyze/alerts-cases/watcher/actions-pagerduty.md @@ -28,9 +28,9 @@ The following snippet shows a simple PagerDuty action definition: } } ``` -% NOTCONSOLE 1. Description of the message +% NOTCONSOLE ## Adding meta information to a PagerDuty incident [adding-context-and-payloads-to-pagerduty-actions] diff --git a/explore-analyze/alerts-cases/watcher/actions-slack.md b/explore-analyze/alerts-cases/watcher/actions-slack.md index 9191c80db6..e7cfc66910 100644 --- a/explore-analyze/alerts-cases/watcher/actions-slack.md +++ b/explore-analyze/alerts-cases/watcher/actions-slack.md @@ -31,10 +31,9 @@ The following snippet shows a simple slack action definition: } } ``` -% NOTCONSOLE - 1. The channels and users you want to send the message to. 2. The content of the message. +% NOTCONSOLE ## Using attachments to format Slack messages [formatting-slack-messages] @@ -121,10 +120,10 @@ In the following example, the watch input executes a search with a date histogra } } ``` -% NOTCONSOLE 1. The list generated by the action’s transform. 2. The parameter placeholders refer to attributes in each item of the list generated by the transform. +% NOTCONSOLE ## Slack action attributes [slack-action-attributes] diff --git a/explore-analyze/alerts-cases/watcher/actions-webhook.md b/explore-analyze/alerts-cases/watcher/actions-webhook.md index 2a0725d787..af1d37f2e7 100644 --- a/explore-analyze/alerts-cases/watcher/actions-webhook.md +++ b/explore-analyze/alerts-cases/watcher/actions-webhook.md @@ -32,7 +32,6 @@ The following snippet shows a simple webhook action definition: } } ``` -% NOTCONSOLE 1. The id of the action 2. An optional [transform](transform.md) to transform the payload before executing the `webhook` action @@ -42,6 +41,7 @@ The following snippet shows a simple webhook action definition: 6. The port to connect to 7. The path (URI) to use in the HTTP request 8. The body to send with the request +% NOTCONSOLE You can use basic authentication when sending a request to a secured webservice. For example, the following `webhook` action creates a new issue in GitHub: @@ -65,9 +65,9 @@ You can use basic authentication when sending a request to a secured webservice. } } ``` -% NOTCONSOLE 1. The username and password for the user creating the issue +% NOTCONSOLE ::::{note} By default, both the username and the password are stored in the `.watches` index in plain text. When the {{es}} {{security-features}} are enabled, {{watcher}} can encrypt the password before storing it. @@ -94,9 +94,9 @@ You can specify query parameters to send with the request with the `params` fiel } } ``` -% NOTCONSOLE 1. The parameter values can contain templated strings. +% NOTCONSOLE ## Custom request headers [webhook-custom-request-headers] @@ -118,9 +118,9 @@ You can specify request headers to send with the request with the `headers` fiel } } ``` -% NOTCONSOLE 1. The header values can contain templated strings. +% NOTCONSOLE ## Webhook action attributes [_webhook_action_attributes] diff --git a/explore-analyze/alerts-cases/watcher/condition-array-compare.md b/explore-analyze/alerts-cases/watcher/condition-array-compare.md index 25282f38bd..b927c033c3 100644 --- a/explore-analyze/alerts-cases/watcher/condition-array-compare.md +++ b/explore-analyze/alerts-cases/watcher/condition-array-compare.md @@ -31,12 +31,12 @@ For example, the following `array_compare` condition returns `true` if there is } } ``` -% NOTCONSOLE 1. The path to the array in the execution context that you want to evaluate, specified in dot notation. 2. The path to the field in each array element that you want to evaluate. 3. The [comparison operator](condition-compare.md#condition-compare-operators) to use. 4. The comparison value. Supports date math like the [compare condition](condition-compare.md#compare-condition-date-math). +% NOTCONSOLE ::::{note} When using fieldnames that contain a dot this condition will not work, use a [script condition](condition-script.md) instead. diff --git a/explore-analyze/alerts-cases/watcher/condition-compare.md b/explore-analyze/alerts-cases/watcher/condition-compare.md index b01f6a5b9d..fabcf6a2b6 100644 --- a/explore-analyze/alerts-cases/watcher/condition-compare.md +++ b/explore-analyze/alerts-cases/watcher/condition-compare.md @@ -37,10 +37,10 @@ To use the `compare` condition, you specify the value in the execution context t } } ``` -% NOTCONSOLE 1. Use dot notation to reference a value in the execution context. 2. Specify a comparison operator and the value you want to compare against. +% NOTCONSOLE You can also compare two values in the execution context by specifying the compared value as a path of the form of `{{path}}`. For example, the following condition compares the `ctx.payload.aggregations.status.buckets.error.doc_count` to the `ctx.payload.aggregations.handled.buckets.true.doc_count`: diff --git a/explore-analyze/alerts-cases/watcher/how-watcher-works.md b/explore-analyze/alerts-cases/watcher/how-watcher-works.md index 2b676de3c2..c7ddee515c 100644 --- a/explore-analyze/alerts-cases/watcher/how-watcher-works.md +++ b/explore-analyze/alerts-cases/watcher/how-watcher-works.md @@ -205,7 +205,6 @@ The following snippet shows the basic structure of the *Watch Execution Context* "vars" : { ... } <6> } ``` -% NOTCONSOLE 1. Any static metadata specified in the watch definition. 2. The current watch payload. @@ -213,7 +212,7 @@ The following snippet shows the basic structure of the *Watch Execution Context* 4. A timestamp that shows when the watch execution started. 5. Information about the trigger event. For a `schedule` trigger, this consists of the `triggered_time` (when the watch was triggered) and the `scheduled_time` (when the watch was scheduled to be triggered). 6. Dynamic variables that can be set and accessed by different constructs during the execution. These variables are scoped to a single execution (i.e they’re not persisted and can’t be used between different executions of the same watch) - +% NOTCONSOLE ### Using scripts [scripts] diff --git a/explore-analyze/alerts-cases/watcher/input-chain.md b/explore-analyze/alerts-cases/watcher/input-chain.md index ca85c75ec6..4f0646bf67 100644 --- a/explore-analyze/alerts-cases/watcher/input-chain.md +++ b/explore-analyze/alerts-cases/watcher/input-chain.md @@ -39,10 +39,10 @@ For example, the following chain input loads data from an HTTP server using the } } ``` -% NOTCONSOLE 1. The inputs in a chain are specified as an array to guarantee the order in which the inputs are processed. (JSON does not guarantee the order of arbitrary objects.) 2. Loads the `path` set by the `first` input. +% NOTCONSOLE ## Accessing chained input data [_accessing_chained_input_data] diff --git a/explore-analyze/alerts-cases/watcher/input-http.md b/explore-analyze/alerts-cases/watcher/input-http.md index a296132fcc..32a5f1bdaf 100644 --- a/explore-analyze/alerts-cases/watcher/input-http.md +++ b/explore-analyze/alerts-cases/watcher/input-http.md @@ -68,9 +68,9 @@ To load the data from other Elasticsearch APIs, specify the API endpoint as the } } ``` -% NOTCONSOLE 1. Enabling this attribute returns the `bytes` values in the response in human readable format. +% NOTCONSOLE ## Calling external web services [input-http-auth-basic-example] diff --git a/explore-analyze/alerts-cases/watcher/transform-chain.md b/explore-analyze/alerts-cases/watcher/transform-chain.md index 4c0036dcfe..e310574ddc 100644 --- a/explore-analyze/alerts-cases/watcher/transform-chain.md +++ b/explore-analyze/alerts-cases/watcher/transform-chain.md @@ -35,10 +35,10 @@ You can use chain {{watcher-transforms}} to build more complex transforms out of ] } ``` -% NOTCONSOLE 1. The `chain` {{watcher-transform}} definition 2. The first transform in the chain (in this case, a `search` {{watcher-transform}}) 3. The second and final transform in the chain (in this case, a `script` {{watcher-transform}}) +% NOTCONSOLE This example executes a `count` search on the cluster to look for `error` events. The search results are then passed to the second `script` {{watcher-transform}}. The `script` {{watcher-transform}} extracts the total hit count and assigns it to the `error_count` field in a newly-generated payload. This new payload is the output of the `chain` {{watcher-transform}} and replaces the payload in the watch execution context. diff --git a/explore-analyze/alerts-cases/watcher/transform-script.md b/explore-analyze/alerts-cases/watcher/transform-script.md index a0d1286ab3..8a2d7c55cc 100644 --- a/explore-analyze/alerts-cases/watcher/transform-script.md +++ b/explore-analyze/alerts-cases/watcher/transform-script.md @@ -22,9 +22,9 @@ The `script` {{watcher-transform}} is often useful when used in combination with } } ``` -% NOTCONSOLE 1. A simple `painless` script that creates a new payload with a single `time` field holding the scheduled time. +% NOTCONSOLE ::::{note} The executed script may either return a valid model that is the equivalent of a Java™ Map or a JSON object (you will need to consult the documentation of the specific scripting language to find out what this construct is). Any other value that is returned will be assigned and accessible to/via the `_value` variable. diff --git a/explore-analyze/alerts-cases/watcher/transform.md b/explore-analyze/alerts-cases/watcher/transform.md index 105a93597e..dd930ac602 100644 --- a/explore-analyze/alerts-cases/watcher/transform.md +++ b/explore-analyze/alerts-cases/watcher/transform.md @@ -54,7 +54,7 @@ The following example defines two {{watcher-transforms}}, one at the watch level ] } ``` -% NOTCONSOLE 1. A watch level `transform` 2. An action level `transform` +% NOTCONSOLE \ No newline at end of file diff --git a/explore-analyze/elastic-inference/inference-api/elasticsearch-inference-integration.md b/explore-analyze/elastic-inference/inference-api/elasticsearch-inference-integration.md index 49f584a0a0..699eae44d6 100644 --- a/explore-analyze/elastic-inference/inference-api/elasticsearch-inference-integration.md +++ b/explore-analyze/elastic-inference/inference-api/elasticsearch-inference-integration.md @@ -128,11 +128,10 @@ PUT _inference/sparse_embedding/my-elser-model } } ``` -% TEST[skip:TBD] 1. Adaptive allocations will be enabled with the minimum of 1 and the maximum of 10 allocations. 2. The `model_id` must be the ID of one of the built-in ELSER models. Valid values are `.elser_model_2` and `.elser_model_2_linux-x86_64`. For further details, refer to the [ELSER model documentation](../../../explore-analyze/machine-learning/nlp/ml-nlp-elser.md). - +% TEST[skip:TBD] ## Elastic Rerank via the `elasticsearch` service [inference-example-elastic-reranker] @@ -162,11 +161,10 @@ PUT _inference/rerank/my-elastic-rerank } } ``` -% TEST[skip:TBD] 1. The `model_id` must be the ID of the built-in Elastic Rerank model: `.rerank-v1`. 2. [Adaptive allocations](../../../deploy-manage/autoscaling/trained-model-autoscaling.md#enabling-autoscaling-through-apis-adaptive-allocations) will be enabled with the minimum of 1 and the maximum of 10 allocations. - +% TEST[skip:TBD] ## E5 via the `elasticsearch` service [inference-example-elasticsearch] @@ -186,10 +184,9 @@ PUT _inference/text_embedding/my-e5-model } } ``` -% TEST[skip:TBD] 1. The `model_id` must be the ID of one of the built-in E5 models. Valid values are `.multilingual-e5-small` and `.multilingual-e5-small_linux-x86_64`. For further details, refer to the [E5 model documentation](../../../explore-analyze/machine-learning/nlp/ml-nlp-e5.md). - +% TEST[skip:TBD] ::::{note} You might see a 502 bad gateway error in the response when using the {{kib}} Console. This error usually just reflects a timeout, while the model downloads in the background. You can check the download progress in the {{ml-app}} UI. If using the Python client, you can set the `timeout` parameter to a higher value. @@ -213,11 +210,10 @@ PUT _inference/text_embedding/my-msmarco-minilm-model <1> } } ``` -% TEST[skip:TBD] 1. Provide an unique identifier for the inference endpoint. The `inference_id` must be unique and must not match the `model_id`. 2. The `model_id` must be the ID of a text embedding model which has already been [uploaded through Eland](../../../explore-analyze/machine-learning/nlp/ml-nlp-import-model.md#ml-nlp-import-script). - +% TEST[skip:TBD] ## Setting adaptive allocation for E5 via the `elasticsearch` service [inference-example-adaptive-allocation] @@ -257,10 +253,9 @@ PUT _inference/sparse_embedding/use_existing_deployment } } ``` -% TEST[skip:TBD] 1. The `deployment_id` of the already existing model deployment. - +% TEST[skip:TBD] The API response contains the `model_id`, and the threads and allocations settings from the model deployment: diff --git a/explore-analyze/elastic-inference/inference-api/huggingface-inference-integration.md b/explore-analyze/elastic-inference/inference-api/huggingface-inference-integration.md index 6dccefba2b..d9da04666a 100644 --- a/explore-analyze/elastic-inference/inference-api/huggingface-inference-integration.md +++ b/explore-analyze/elastic-inference/inference-api/huggingface-inference-integration.md @@ -97,11 +97,10 @@ PUT _inference/text_embedding/hugging-face-embeddings } } ``` -% TEST[skip:TBD] 1. A valid Hugging Face access token. You can find on the [settings page of your account](https://huggingface.co/settings/tokens). 2. The {{infer}} endpoint URL you created on Hugging Face. - +% TEST[skip:TBD] Create a new {{infer}} endpoint on [the Hugging Face endpoint page](https://ui.endpoints.huggingface.co/) to get an endpoint URL. Select the model you want to use on the new endpoint creation page - for example `intfloat/e5-small-v2` - then select the `Sentence Embeddings` task under the Advanced configuration section. Create the endpoint. Copy the URL after the endpoint initialization has been finished. diff --git a/explore-analyze/elastic-inference/inference-api/mistral-inference-integration.md b/explore-analyze/elastic-inference/inference-api/mistral-inference-integration.md index fa71c2d2e1..8cf7f35333 100644 --- a/explore-analyze/elastic-inference/inference-api/mistral-inference-integration.md +++ b/explore-analyze/elastic-inference/inference-api/mistral-inference-integration.md @@ -100,8 +100,7 @@ PUT _inference/text_embedding/mistral-embeddings-test } } ``` -% TEST[skip:TBD] 1. The `model` must be the ID of a text embedding model which can be found in the [Mistral models documentation](https://docs.mistral.ai/getting-started/models/). - +% TEST[skip:TBD] diff --git a/explore-analyze/query-filter/aggregations.md b/explore-analyze/query-filter/aggregations.md index f1ab09173f..c28e41ac58 100644 --- a/explore-analyze/query-filter/aggregations.md +++ b/explore-analyze/query-filter/aggregations.md @@ -69,13 +69,13 @@ Aggregation results are in the response’s `aggregations` object: } } ``` + +1. Results for the `my-agg-name` aggregation. % TESTRESPONSE[s/"took": 78/"took": "$body.took"/] % TESTRESPONSE[s/...$/"took": "$body.took", "timed_out": false, "_shards": "$body._shards", /] % TESTRESPONSE[s/"hits": [...]/"hits": "$body.hits.hits"/] % TESTRESPONSE[s/"buckets": []/"buckets":[{"key":"get","doc_count":5}]/] -1. Results for the `my-agg-name` aggregation. - ## Change an aggregation’s scope [change-agg-scope] Use the `query` parameter to limit the documents on which an aggregation runs: @@ -198,12 +198,12 @@ The response nests sub-aggregation results under their parent aggregation: } } ``` -% TESTRESPONSE[s/.../"took": "$body.took", "timed_out": false, "_shards": "$body._shards", "hits": "$body.hits",/] -% TESTRESPONSE[s/"key": "foo"/"key": "get"/] -% TESTRESPONSE[s/"value": 75.0/"value": $body.aggregations.my-agg-name.buckets.0.my-sub-agg-name.value/] 1. Results for the parent aggregation, `my-agg-name`. 2. Results for `my-agg-name`'s sub-aggregation, `my-sub-agg-name`. +% TESTRESPONSE[s/.../"took": "$body.took", "timed_out": false, "_shards": "$body._shards", "hits": "$body.hits",/] +% TESTRESPONSE[s/"key": "foo"/"key": "get"/] +% TESTRESPONSE[s/"value": 75.0/"value": $body.aggregations.my-agg-name.buckets.0.my-sub-agg-name.value/] ## Add custom metadata [add-metadata-to-an-agg] @@ -283,10 +283,10 @@ Some aggregations return a different aggregation type from the type in the reque } } ``` -% TESTRESPONSE[s/.../"took": "$body.took", "timed_out": false, "_shards": "$body._shards", "hits": "$body.hits",/] -% TESTRESPONSE[s/"buckets": []/"buckets":[{"key":1070000.0,"doc_count":5}]/] 1. The aggregation type, `histogram`, followed by a `#` separator and the aggregation’s name, `my-agg-name`. +% TESTRESPONSE[s/.../"took": "$body.took", "timed_out": false, "_shards": "$body._shards", "hits": "$body.hits",/] +% TESTRESPONSE[s/"buckets": []/"buckets":[{"key":1070000.0,"doc_count":5}]/] ## Use scripts in an aggregation [use-scripts-in-an-agg] diff --git a/explore-analyze/query-filter/aggregations/tutorial-analyze-ecommerce-data-with-aggregations-using-query-dsl.md b/explore-analyze/query-filter/aggregations/tutorial-analyze-ecommerce-data-with-aggregations-using-query-dsl.md index bb77e1c249..db8fee6fee 100644 --- a/explore-analyze/query-filter/aggregations/tutorial-analyze-ecommerce-data-with-aggregations-using-query-dsl.md +++ b/explore-analyze/query-filter/aggregations/tutorial-analyze-ecommerce-data-with-aggregations-using-query-dsl.md @@ -306,11 +306,11 @@ GET kibana_sample_data_ecommerce/_search } } ``` -% TEST[skip:Using Kibana sample data] 1. Set `size` to 0 to avoid returning matched documents in the response and return only the aggregation results 2. A meaningful name that describes what this metric represents 3. Configures an `avg` aggregation, which calculates a simple arithmetic mean +% TEST[skip:Using Kibana sample data] ::::{dropdown} Example response @@ -339,12 +339,12 @@ GET kibana_sample_data_ecommerce/_search } } ``` -% TEST[skip:Using Kibana sample data] 1. Total number of orders in the dataset 2. `hits` is empty because we set `size` to 0 3. Results appear under the name we specified in the request 4. The average order value is calculated dynamically from all the orders in the dataset +% TEST[skip:Using Kibana sample data] :::: @@ -365,10 +365,10 @@ GET kibana_sample_data_ecommerce/_search } } ``` -% TEST[skip:Using Kibana sample data] 1. A descriptive name for this set of statistics 2. `stats` returns count, min, max, avg, and sum at once +% TEST[skip:Using Kibana sample data] ::::{dropdown} Example response @@ -385,13 +385,13 @@ GET kibana_sample_data_ecommerce/_search } } ``` -% TEST[skip:Using Kibana sample data] 1. `"count"`: Total number of orders in the dataset 2. `"min"`: Lowest individual order value in the dataset 3. `"max"`: Highest individual order value in the dataset 4. `"avg"`: Average value per order across all orders 5. `"sum"`: Total revenue from all orders combined +% TEST[skip:Using Kibana sample data] :::: @@ -423,13 +423,13 @@ GET kibana_sample_data_ecommerce/_search } } ``` -% TEST[skip:Using Kibana sample data] 1. Name reflecting the business purpose of this breakdown 2. `terms` aggregation groups documents by field values 3. Use [`.keyword`](elasticsearch://reference/elasticsearch/mapping-reference/keyword.md) field for exact matching on text fields 4. Limit to top 5 categories 5. Order by number of orders (descending) +% TEST[skip:Using Kibana sample data] ::::{dropdown} Example response @@ -481,13 +481,13 @@ GET kibana_sample_data_ecommerce/_search } } ``` -% TEST[skip:Using Kibana sample data] 1. Due to Elasticsearch’s distributed architecture, when [terms aggregations](elasticsearch://reference/data-analysis/aggregations/search-aggregations-bucket-terms-aggregation.md) run across multiple shards, the doc counts may have a small margin of error. This value indicates the maximum possible error in the counts. 2. Count of documents in categories beyond the requested size. 3. Array of category buckets, ordered by count. 4. Category name. 5. Number of orders in this category. +% TEST[skip:Using Kibana sample data] :::: @@ -511,13 +511,13 @@ GET kibana_sample_data_ecommerce/_search } } ``` -% TEST[skip:Using Kibana sample data] 1. Descriptive name for the time-series aggregation results. 2. The `date_histogram` aggregation groups documents into time-based buckets, similar to terms aggregation but for dates. 3. Uses [calendar and fixed time intervals](elasticsearch://reference/data-analysis/aggregations/search-aggregations-bucket-datehistogram-aggregation.md#calendar_and_fixed_intervals) to handle months with different lengths. `"day"` ensures consistent daily grouping regardless of timezone. 4. Formats dates in response using [date patterns](elasticsearch://reference/elasticsearch/mapping-reference/mapping-date-format.md) (e.g. "yyyy-MM-dd"). Refer to [date math expressions](elasticsearch://reference/elasticsearch/rest-apis/common-options.md#date-math) for additional options. 5. When `min_doc_count` is 0, returns buckets for days with no orders, useful for continuous time series visualization. +% TEST[skip:Using Kibana sample data] ::::{dropdown} Example response @@ -702,13 +702,13 @@ GET kibana_sample_data_ecommerce/_search } } ``` -% TEST[skip:Using Kibana sample data] 1. Results of our named aggregation "daily_orders" 2. Time-based buckets from date_histogram aggregation 3. `key_as_string` is the human-readable date for this bucket 4. `key` is the same date represented as the Unix timestamp for this bucket 5. `doc_count` counts the number of documents that fall into this time bucket +% TEST[skip:Using Kibana sample data] :::: @@ -752,13 +752,13 @@ GET kibana_sample_data_ecommerce/_search } } ``` -% TEST[skip:Using Kibana sample data] 1. Order categories by their total revenue instead of count 2. Define metrics to calculate within each category 3. Total revenue for the category 4. Average order value in the category 5. Total number of items sold +% TEST[skip:Using Kibana sample data] ::::{dropdown} Example response @@ -790,13 +790,13 @@ GET kibana_sample_data_ecommerce/_search } } ``` -% TEST[skip:Using Kibana sample data] 1. Category name 2. Number of orders 3. Total revenue for this category 4. Average order value for this category 5. Total quantity of items sold +% TEST[skip:Using Kibana sample data] :::: @@ -836,11 +836,11 @@ GET kibana_sample_data_ecommerce/_search } } ``` -% TEST[skip:Using Kibana sample data] 1. Daily revenue 2. Uses the [`cardinality`](elasticsearch://reference/data-analysis/aggregations/search-aggregations-metrics-cardinality-aggregation.md) aggregation to count unique customers per day 3. Average number of items per order +% TEST[skip:Using Kibana sample data] ::::{dropdown} Example response @@ -1343,7 +1343,6 @@ GET kibana_sample_data_ecommerce/_search } } ``` -% TEST[skip:Using Kibana sample data] 1. Calculate daily revenue first. 2. Create a smoothed version of the daily revenue. @@ -1351,6 +1350,7 @@ GET kibana_sample_data_ecommerce/_search 4. Reference the revenue from our date histogram. 5. Use a 3-day window — use different window sizes to see trends at different time scales. 6. Use the built-in unweighted average function in the `moving_fn` aggregation. +% TEST[skip:Using Kibana sample data] ::::{dropdown} Example response @@ -1721,13 +1721,13 @@ GET kibana_sample_data_ecommerce/_search } } ``` -% TEST[skip:Using Kibana sample data] 1. Date of the bucket is in default ISO format because we didn’t specify a format 2. Number of orders for this day 3. Raw daily revenue before smoothing 4. First day has no smoothed value as it needs previous days for the calculation 5. Moving average starts from second day, using a 3-day window +% TEST[skip:Using Kibana sample data] :::: @@ -1766,11 +1766,11 @@ GET kibana_sample_data_ecommerce/_search } } ``` -% TEST[skip:Using Kibana sample data] 1. Name for our running total 2. `cumulative_sum` adds up values across buckets 3. Reference the revenue we want to accumulate +% TEST[skip:Using Kibana sample data] ::::{dropdown} Example response @@ -2141,13 +2141,13 @@ GET kibana_sample_data_ecommerce/_search } } ``` -% TEST[skip:Using Kibana sample data] 1. `daily_sales`: Results from our daily sales date histogram 2. `buckets`: Array of time-based buckets 3. `key_as_string`: Date for this bucket (in ISO format since no format specified) 4. `revenue`: Daily revenue for this date 5. `cumulative_revenue`: Running total of revenue up to this date +% TEST[skip:Using Kibana sample data] :::: diff --git a/explore-analyze/query-filter/languages/esql-cross-clusters.md b/explore-analyze/query-filter/languages/esql-cross-clusters.md index 4231e3dd3c..15a2ac6d14 100644 --- a/explore-analyze/query-filter/languages/esql-cross-clusters.md +++ b/explore-analyze/query-filter/languages/esql-cross-clusters.md @@ -168,11 +168,10 @@ PUT _cluster/settings } } ``` -% TEST[setup:host] -% TEST[s/35.238.149.\d+:930\d+/${transport_host}/] 1. Since `skip_unavailable` was not set on `cluster_three`, it uses the default of `false`. See the [Optional remote clusters](#ccq-skip-unavailable-clusters) section for details. - +% TEST[setup:host] +% TEST[s/35.238.149.\d+:930\d+/${transport_host}/] ## Query across multiple clusters [ccq-from] @@ -286,7 +285,6 @@ Which returns: } } ``` -% TEST[skip: cross-cluster testing env not set up] 1. How long the entire search (across all clusters) took, in milliseconds. 2. This section of counters shows all possible cluster search states and how many cluster searches are currently in that state. The clusters can have one of the following statuses: **running**, **successful** (searches on all shards were successful), **skipped** (the search failed on a cluster marked with `skip_unavailable`=`true`), **failed** (the search failed on a cluster marked with `skip_unavailable`=`false`) or **partial** (the search was [interrupted](https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-async-query-stop-api.html) before finishing). @@ -295,7 +293,7 @@ Which returns: 5. How long (in milliseconds) the search took on each cluster. This can be useful to determine which clusters have slower response times than others. 6. The shard details for the search on that cluster, including a count of shards that were skipped due to the can-match phase results. Shards are skipped when they cannot have any matching data and therefore are not included in the full ES|QL query. 7. The `is_partial` field is set to `true` if the search has partial results for any reason, for example if it was interrupted before finishing using the [async query stop API](https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-async-query-stop-api.html). - +% TEST[skip: cross-cluster testing env not set up] The cross-cluster metadata can be used to determine whether any data came back from a cluster. For instance, in the query below, the wildcard expression for `cluster-two` did not resolve to a concrete index (or indices). The cluster is, therefore, marked as *skipped* and the total number of shards searched is set to zero. diff --git a/explore-analyze/query-filter/languages/esql-task-management.md b/explore-analyze/query-filter/languages/esql-task-management.md index c34fd12e63..0e96c27716 100644 --- a/explore-analyze/query-filter/languages/esql-task-management.md +++ b/explore-analyze/query-filter/languages/esql-task-management.md @@ -38,11 +38,10 @@ Which returns a list of statuses like this: "headers" : { } } ``` -% NOTCONSOLE 1. The user submitted query. 2. Time the query has been running. - +% NOTCONSOLE You can use this to find long running queries and, if you need to, cancel them with the [task cancellation API](https://www.elastic.co/docs/api/doc/elasticsearch/group/endpoint-tasks#task-cancellation): diff --git a/explore-analyze/query-filter/languages/example-detect-threats-with-eql.md b/explore-analyze/query-filter/languages/example-detect-threats-with-eql.md index 55be66ab82..979f541247 100644 --- a/explore-analyze/query-filter/languages/example-detect-threats-with-eql.md +++ b/explore-analyze/query-filter/languages/example-detect-threats-with-eql.md @@ -71,12 +71,11 @@ GET /my-data-stream/_eql/search?filter_path=-hits.events <1> "size": 200 <3> } ``` -% TEST[setup:atomic_red_regsvr32] 1. `?filter_path=-hits.events` excludes the `hits.events` property from the response. This search is only intended to get an event count, not a list of matching events. 2. Matches any event with a `process.name` of `regsvr32.exe`. 3. Returns up to 200 hits for matching events. - +% TEST[setup:atomic_red_regsvr32] The response returns 143 related events. diff --git a/explore-analyze/transforms/transform-examples.md b/explore-analyze/transforms/transform-examples.md index 7b68b2aa07..a0bdb12a46 100644 --- a/explore-analyze/transforms/transform-examples.md +++ b/explore-analyze/transforms/transform-examples.md @@ -11,12 +11,13 @@ mapped_pages: These examples demonstrate how to use {{transforms}} to derive useful insights from your data. All the examples use one of the [{{kib}} sample datasets](/explore-analyze/index.md). For a more detailed, step-by-step example, see [Tutorial: Transforming the eCommerce sample data](ecommerce-transforms.md). -* [Finding your best customers](#example-best-customers) -* [Finding air carriers with the most delays](#example-airline) -* [Finding suspicious client IPs](#example-clientips) -* [Finding the last log event for each IP address](#example-last-log) -* [Finding client IPs that sent the most bytes to the server](#example-bytes) -* [Getting customer name and email address by customer ID](#example-customer-names) +- [Examples \[transform-examples\]](#examples-transform-examples) + - [Finding your best customers \[example-best-customers\]](#finding-your-best-customers-example-best-customers) + - [Finding air carriers with the most delays \[example-airline\]](#finding-air-carriers-with-the-most-delays-example-airline) + - [Finding suspicious client IPs \[example-clientips\]](#finding-suspicious-client-ips-example-clientips) + - [Finding the last log event for each IP address \[example-last-log\]](#finding-the-last-log-event-for-each-ip-address-example-last-log) + - [Finding client IPs that sent the most bytes to the server \[example-bytes\]](#finding-client-ips-that-sent-the-most-bytes-to-the-server-example-bytes) + - [Getting customer name and email address by customer ID \[example-customer-names\]](#getting-customer-name-and-email-address-by-customer-id-example-customer-names) ## Finding your best customers [example-best-customers] @@ -54,10 +55,10 @@ POST _transform/_preview } } ``` -% TEST[skip:setup kibana sample data] 1. The destination index for the {{transform}}. It is ignored by `_preview`. 2. Two `group_by` fields is selected. This means the {{transform}} contains a unique row per `user` and `customer_id` combination. Within this data set, both these fields are unique. By including both in the {{transform}}, it gives more context to the final results. +% TEST[skip:setup kibana sample data] ::::{note} In the example above, condensed JSON formatting is used for easier readability of the pivot object. @@ -135,12 +136,12 @@ POST _transform/_preview } } ``` -% TEST[skip:setup kibana sample data] 1. Filter the source data to select only flights that are not cancelled. 2. The destination index for the {{transform}}. It is ignored by `_preview`. 3. The data is grouped by the `Carrier` field which contains the airline name. 4. This `bucket_script` performs calculations on the results that are returned by the aggregation. In this particular example, it calculates what percentage of travel time was taken up by delays. +% TEST[skip:setup kibana sample data] The preview shows you that the new index would contain data like this for each carrier: @@ -225,13 +226,13 @@ PUT _transform/suspicious_client_ips } } ``` -% TEST[skip:setup kibana sample data] 1. The destination index for the {{transform}}. 2. Configures the {{transform}} to run continuously. It uses the `timestamp` field to synchronize the source and destination indices. The worst case ingestion delay is 60 seconds. 3. The data is grouped by the `clientip` field. 4. Filter aggregation that counts the occurrences of successful (`200`) responses in the `response` field. The following two aggregations (`error404` and `error5xx`) count the error responses by error codes, matching an exact value or a range of response codes. 5. This `bucket_script` calculates the duration of the `clientip` access based on the results of the aggregation. +% TEST[skip:setup kibana sample data] After you create the {{transform}}, you must start it: @@ -348,13 +349,13 @@ PUT _transform/last-log-from-clientip } } ``` -% TEST[skip:setup kibana sample data] 1. Specifies the field for grouping the data. 2. Specifies the date field that is used for sorting the data. 3. Sets the interval for the {{transform}} to check for changes in the source index. 4. Contains the time field and delay settings used to synchronize the source and destination indices. 5. Specifies the retention policy for the transform. Documents that are older than the configured value will be removed from the destination index. +% TEST[skip:setup kibana sample data] After you create the {{transform}}, start it: @@ -471,11 +472,11 @@ POST _transform/_preview } } ``` -% TEST[skip:setup kibana sample data] 1. The data is grouped by a date histogram of the time field with a one hour interval. 2. Calculates the maximum value of the `bytes` field. 3. Specifies the fields (`clientip` and `geo.src`) of the top document to return and the sorting method (document with the highest `bytes` value). +% TEST[skip:setup kibana sample data] The API call above returns a response similar to this: @@ -561,10 +562,10 @@ POST _transform/_preview } } ``` -% TEST[skip:setup kibana sample data] 1. The data is grouped by a `terms` aggregation on the `customer_id` field. 2. Specifies the fields to return (email and name fields) in a descending order by the order date. +% TEST[skip:setup kibana sample data] The API returns a response that is similar to this: diff --git a/explore-analyze/transforms/transform-painless-examples.md b/explore-analyze/transforms/transform-painless-examples.md index ab570f05a3..8c59f3d5ac 100644 --- a/explore-analyze/transforms/transform-painless-examples.md +++ b/explore-analyze/transforms/transform-painless-examples.md @@ -15,12 +15,14 @@ The examples that use the `scripted_metric` aggregation are not supported on {{e These examples demonstrate how to use Painless in {{transforms}}. You can learn more about the Painless scripting language in the [Painless guide](elasticsearch://reference/scripting-languages/painless/painless.md). -* [Getting top hits by using scripted metric aggregation](#painless-top-hits) -* [Getting time features by using aggregations](#painless-time-features) -* [Getting duration by using bucket script](#painless-bucket-script) -* [Counting HTTP responses by using scripted metric aggregation](#painless-count-http) -* [Comparing indices by using scripted metric aggregations](#painless-compare) -* [Getting web session details by using scripted metric aggregation](#painless-web-session) +- [Painless examples \[transform-painless-examples\]](#painless-examples-transform-painless-examples) + - [Getting top hits by using scripted metric aggregation \[painless-top-hits\]](#getting-top-hits-by-using-scripted-metric-aggregation-painless-top-hits) + - [Getting top hits by using stored scripts \[top-hits-stored-scripts\]](#getting-top-hits-by-using-stored-scripts-top-hits-stored-scripts) + - [Getting time features by using aggregations \[painless-time-features\]](#getting-time-features-by-using-aggregations-painless-time-features) + - [Getting duration by using bucket script \[painless-bucket-script\]](#getting-duration-by-using-bucket-script-painless-bucket-script) + - [Counting HTTP responses by using scripted metric aggregation \[painless-count-http\]](#counting-http-responses-by-using-scripted-metric-aggregation-painless-count-http) + - [Comparing indices by using scripted metric aggregations \[painless-compare\]](#comparing-indices-by-using-scripted-metric-aggregations-painless-compare) + - [Getting web session details by using scripted metric aggregation \[painless-web-session\]](#getting-web-session-details-by-using-scripted-metric-aggregation-painless-web-session) ::::{note} @@ -60,12 +62,12 @@ This example uses a `scripted_metric` aggregation which is not supported on {{es } } ``` -% NOTCONSOLE 1. The `init_script` creates a long type `timestamp_latest` and a string type `last_doc` in the `state` object. 2. The `map_script` defines `current_date` based on the timestamp of the document, then compares `current_date` with `state.timestamp_latest`, finally returns `state.last_doc` from the shard. By using `new HashMap(...)` you copy the source document, this is important whenever you want to pass the full source object from one phase to the next. 3. The `combine_script` returns `state` from each shard. 4. The `reduce_script` iterates through the value of `s.timestamp_latest` returned by each shard and returns the document with the latest timestamp (`last_doc`). In the response, the top hit (in other words, the `latest_doc`) is nested below the `latest_doc` field. +% NOTCONSOLE Check the [scope of scripts](elasticsearch://reference/data-analysis/aggregations/search-aggregations-metrics-scripted-metric-aggregation.md#scripted-metric-aggregation-scope) for detailed explanation on the respective scripts. @@ -173,10 +175,10 @@ You can also use the power of [stored scripts](https://www.elastic.co/docs/api/d "id":"last-value-reduce" } ``` - % NOTCONSOLE 1. The parameter `field_with_last_value` can be set any field that you want the latest value for. - + % NOTCONSOLE + ## Getting time features by using aggregations [painless-time-features] This snippet shows how to extract time based features by using Painless in a {{transform}}. The snippet uses an index where `@timestamp` is defined as a `date` type field. diff --git a/manage-data/data-store/data-streams/logs-data-stream.md b/manage-data/data-store/data-streams/logs-data-stream.md index 44b5dc6640..2655ebc5db 100644 --- a/manage-data/data-store/data-streams/logs-data-stream.md +++ b/manage-data/data-store/data-streams/logs-data-stream.md @@ -35,11 +35,10 @@ PUT _index_template/my-index-template "priority": 101 <2> } ``` -% TEST 1. The index mode setting. 2. The index template priority. By default, Elasticsearch ships with a `logs-*-*` index template with a priority of 100. To make sure your index template takes priority over the default `logs-*-*` template, set its `priority` to a number higher than 100. For more information, see [Avoid index pattern collisions](../templates.md#avoid-index-pattern-collisions). - +% TEST After the index template is created, new indices that use the template will be configured as a logs data stream. You can start indexing data and [using the data stream](use-data-stream.md). diff --git a/manage-data/data-store/data-streams/run-downsampling-manually.md b/manage-data/data-store/data-streams/run-downsampling-manually.md index 960d66adf2..4acdbc876c 100644 --- a/manage-data/data-store/data-streams/run-downsampling-manually.md +++ b/manage-data/data-store/data-streams/run-downsampling-manually.md @@ -352,15 +352,14 @@ This returns: ] } ``` + +1. The backing index for this data stream. % TESTRESPONSE[s/".ds-my-data-stream-2023.07.26-000001"/$body.data_streams.0.indices.0.index_name/] % TESTRESPONSE[s/"ltOJGmqgTVm4T-Buoe7Acg"/$body.data_streams.0.indices.0.index_uuid/] % TESTRESPONSE[s/"2023-07-26T09:26:42.000Z"/$body.data_streams.0.time_series.temporal_ranges.0.start/] % TESTRESPONSE[s/"2023-07-26T13:26:42.000Z"/$body.data_streams.0.time_series.temporal_ranges.0.end/] % TESTRESPONSE[s/"replicated": false/"replicated": false,"failure_store":{"enabled": false, "indices": [], "rollover_on_write": true}/] -1. The backing index for this data stream. - - Before a backing index can be downsampled, the TSDS needs to be rolled over and the old index needs to be made read-only. Roll over the TSDS using the [rollover API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-rollover): diff --git a/manage-data/data-store/data-streams/use-data-stream.md b/manage-data/data-store/data-streams/use-data-stream.md index 0ffbc39822..308e3152df 100644 --- a/manage-data/data-store/data-streams/use-data-stream.md +++ b/manage-data/data-store/data-streams/use-data-stream.md @@ -10,15 +10,16 @@ applies_to: After you [set up a data stream](set-up-data-stream.md), you can do the following: -* [Add documents to a data stream](#add-documents-to-a-data-stream) -* [Search a data stream](#search-a-data-stream) -* [Get statistics for a data stream](#get-stats-for-a-data-stream) -* [Manually roll over a data stream](#manually-roll-over-a-data-stream) -* [Open closed backing indices](#open-closed-backing-indices) -* [Reindex with a data stream](#reindex-with-a-data-stream) -* [Update documents in a data stream by query](#update-docs-in-a-data-stream-by-query) -* [Delete documents in a data stream by query](#delete-docs-in-a-data-stream-by-query) -* [Update or delete documents in a backing index](#update-delete-docs-in-a-backing-index) +- [Use a data stream \[use-a-data-stream\]](#use-a-data-stream-use-a-data-stream) + - [Add documents to a data stream \[add-documents-to-a-data-stream\]](#add-documents-to-a-data-stream-add-documents-to-a-data-stream) + - [Search a data stream \[search-a-data-stream\]](#search-a-data-stream-search-a-data-stream) + - [Get statistics for a data stream \[get-stats-for-a-data-stream\]](#get-statistics-for-a-data-stream-get-stats-for-a-data-stream) + - [Manually roll over a data stream \[manually-roll-over-a-data-stream\]](#manually-roll-over-a-data-stream-manually-roll-over-a-data-stream) + - [Open closed backing indices \[open-closed-backing-indices\]](#open-closed-backing-indices-open-closed-backing-indices) + - [Reindex with a data stream \[reindex-with-a-data-stream\]](#reindex-with-a-data-stream-reindex-with-a-data-stream) + - [Update documents in a data stream by query \[update-docs-in-a-data-stream-by-query\]](#update-documents-in-a-data-stream-by-query-update-docs-in-a-data-stream-by-query) + - [Delete documents in a data stream by query \[delete-docs-in-a-data-stream-by-query\]](#delete-documents-in-a-data-stream-by-query-delete-docs-in-a-data-stream-by-query) + - [Update or delete documents in a backing index \[update-delete-docs-in-a-backing-index\]](#update-or-delete-documents-in-a-backing-index-update-delete-docs-in-a-backing-index) ## Add documents to a data stream [add-documents-to-a-data-stream] @@ -226,16 +227,15 @@ Response: } } ``` -% TESTRESPONSE[s/"took": 20/"took": $body.took/] -% TESTRESPONSE[s/"max_score": 0.2876821/"max_score": $body.hits.max_score/] -% TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.03.08-000003"/"_index": $body.hits.hits.0._index/] -% TESTRESPONSE[s/"_score": 0.2876821/"_score": $body.hits.hits.0._score/] 1. Backing index containing the matching document 2. Document ID for the document 3. Current sequence number for the document 4. Primary term for the document - +% TESTRESPONSE[s/"took": 20/"took": $body.took/] +% TESTRESPONSE[s/"max_score": 0.2876821/"max_score": $body.hits.max_score/] +% TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.03.08-000003"/"_index": $body.hits.hits.0._index/] +% TESTRESPONSE[s/"_score": 0.2876821/"_score": $body.hits.hits.0._score/] To update the document, use an [index API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-create) request with valid `if_seq_no` and `if_primary_term` arguments: diff --git a/manage-data/data-store/mapping/dynamic-templates.md b/manage-data/data-store/mapping/dynamic-templates.md index 8f27f0af06..54e9e7bae5 100644 --- a/manage-data/data-store/mapping/dynamic-templates.md +++ b/manage-data/data-store/mapping/dynamic-templates.md @@ -35,12 +35,11 @@ Dynamic templates are specified as an array of named objects: ... ] ``` -% NOTCONSOLE 1. The template name can be any string value. 2. The match conditions can include any of : `match_mapping_type`, `match`, `match_pattern`, `unmatch`, `path_match`, `path_unmatch`. 3. The mapping that the matched field should use. - +% NOTCONSOLE ## Validating dynamic templates [dynamic-templates-validation] diff --git a/manage-data/lifecycle/data-stream/tutorial-migrate-ilm-managed-data-stream-to-data-stream-lifecycle.md b/manage-data/lifecycle/data-stream/tutorial-migrate-ilm-managed-data-stream-to-data-stream-lifecycle.md index 8d6108390f..2e69a6c25b 100644 --- a/manage-data/lifecycle/data-stream/tutorial-migrate-ilm-managed-data-stream-to-data-stream-lifecycle.md +++ b/manage-data/lifecycle/data-stream/tutorial-migrate-ilm-managed-data-stream-to-data-stream-lifecycle.md @@ -129,11 +129,6 @@ Inspecting the response we’ll see that both backing indices are managed by {{i ] } ``` -% TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000001"/"index_name": $body.data_streams.0.indices.0.index_name/] -% TESTRESPONSE[s/"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg"/"index_uuid": $body.data_streams.0.indices.0.index_uuid/] -% TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000002"/"index_name": $body.data_streams.0.indices.1.index_name/] -% TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/] -% TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_store":{"enabled": false, "indices": [], "rollover_on_write": true}/] 1. The name of the backing index. 2. For each backing index we display the value of the [prefer_ilm](elasticsearch://reference/elasticsearch/configuration-reference/data-stream-lifecycle-settings.md#index-lifecycle-prefer-ilm) configuration which will indicate if {{ilm-init}} takes precedence over data stream lifecycle in case both systems are configured for an index. @@ -142,7 +137,11 @@ Inspecting the response we’ll see that both backing indices are managed by {{i 5. The system that will manage the next generation index (the new write index of this data stream, once the data stream is rolled over). The possible values are "Index Lifecycle Management", "Data stream lifecycle", or "Unmanaged". 6. The [prefer_ilm](elasticsearch://reference/elasticsearch/configuration-reference/data-stream-lifecycle-settings.md#index-lifecycle-prefer-ilm) value configured in the index template that’s backing the data stream. This value will be configured for all the new backing indices. If it’s not configured in the index template the backing indices will receive the `true` default value ({{ilm-init}} takes precedence over data stream lifecycle by default as it’s currently richer in features). 7. The {{ilm-init}} policy configured in the index template that’s backing this data stream (which will be configured on all the new backing indices, as long as it exists in the index template). - +% TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000001"/"index_name": $body.data_streams.0.indices.0.index_name/] +% TESTRESPONSE[s/"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg"/"index_uuid": $body.data_streams.0.indices.0.index_uuid/] +% TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000002"/"index_name": $body.data_streams.0.indices.1.index_name/] +% TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/] +% TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_store":{"enabled": false, "indices": [], "rollover_on_write": true}/] ## Migrate data stream to data stream lifecycle [migrate-from-ilm-to-dsl] @@ -177,11 +176,10 @@ PUT _index_template/dsl-data-stream-template } } ``` -% TEST[continued] 1. The `prefer_ilm` setting will now be configured on the **new** backing indices (created by rolling over the data stream) such that {{ilm-init}} does *not* take precedence over data stream lifecycle. 2. We’re configuring the data stream lifecycle so *new* data streams will be managed by data stream lifecycle. - +% TEST[continued] We’ve now made sure that new data streams will be managed by data stream lifecycle. @@ -247,17 +245,16 @@ GET _data_stream/dsl-data-stream ] } ``` -% TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000001"/"index_name": $body.data_streams.0.indices.0.index_name/] -% TESTRESPONSE[s/"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg"/"index_uuid": $body.data_streams.0.indices.0.index_uuid/] -% TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000002"/"index_name": $body.data_streams.0.indices.1.index_name/] -% TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/] -% TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_store":{"enabled": false, "indices": [], "rollover_on_write": true}/] 1. The existing backing index will continue to be managed by {{ilm-init}} 2. The existing backing index will continue to be managed by {{ilm-init}} 3. The next generation index will be managed by Data stream lifecycle 4. The `prefer_ilm` setting value we configured in the index template is reflected and will be configured accordingly for new backing indices. - +% TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000001"/"index_name": $body.data_streams.0.indices.0.index_name/] +% TESTRESPONSE[s/"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg"/"index_uuid": $body.data_streams.0.indices.0.index_uuid/] +% TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000002"/"index_name": $body.data_streams.0.indices.1.index_name/] +% TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/] +% TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_store":{"enabled": false, "indices": [], "rollover_on_write": true}/] We’ll now rollover the data stream to see the new generation index being managed by data stream lifecycle: @@ -323,6 +320,11 @@ GET _data_stream/dsl-data-stream ] } ``` + +1. The backing indices that existed before rollover will continue to be managed by {{ilm-init}} +2. The backing indices that existed before rollover will continue to be managed by {{ilm-init}} +3. The new write index received the `false` value for the `prefer_ilm` setting, as we configured in the index template +4. The new write index is managed by `Data stream lifecycle` % TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000001"/"index_name": $body.data_streams.0.indices.0.index_name/] % TESTRESPONSE[s/"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg"/"index_uuid": $body.data_streams.0.indices.0.index_uuid/] % TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000002"/"index_name": $body.data_streams.0.indices.1.index_name/] @@ -331,12 +333,6 @@ GET _data_stream/dsl-data-stream % TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8abcd1"/"index_uuid": $body.data_streams.0.indices.2.index_uuid/] % TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_store":{"enabled": false, "indices": [], "rollover_on_write": true}/] -1. The backing indices that existed before rollover will continue to be managed by {{ilm-init}} -2. The backing indices that existed before rollover will continue to be managed by {{ilm-init}} -3. The new write index received the `false` value for the `prefer_ilm` setting, as we configured in the index template -4. The new write index is managed by `Data stream lifecycle` - - ## Migrate data stream back to ILM [migrate-from-dsl-to-ilm] @@ -356,10 +352,9 @@ PUT _data_stream/dsl-data-stream/_lifecycle "enabled": false <1> } ``` -% TEST[continued] 1. The `enabled` flag can be ommitted and defaults to `true` however, here we explicitly configure it to `false` Let’s check the state of the data stream: - +% TEST[continued] ```console GET _data_stream/dsl-data-stream @@ -416,6 +411,10 @@ GET _data_stream/dsl-data-stream ] } ``` + +1. The write index is now managed by {{ilm-init}} +2. The `lifecycle` configured on the data stream is now disabled. +3. The next write index will be managed by {{ilm-init}} % TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000001"/"index_name": $body.data_streams.0.indices.0.index_name/] % TESTRESPONSE[s/"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg"/"index_uuid": $body.data_streams.0.indices.0.index_uuid/] % TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000002"/"index_name": $body.data_streams.0.indices.1.index_name/] @@ -424,9 +423,4 @@ GET _data_stream/dsl-data-stream % TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8abcd1"/"index_uuid": $body.data_streams.0.indices.2.index_uuid/] % TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_store":{"enabled": false, "indices": [], "rollover_on_write": true}/] -1. The write index is now managed by {{ilm-init}} -2. The `lifecycle` configured on the data stream is now disabled. -3. The next write index will be managed by {{ilm-init}} - - Had we removed the {{ilm-init}} policy from the index template when we [updated](#update-index-template-for-dsl) it, the write index of the data stream will now be `Unmanaged` because the index wouldn’t have the {{ilm-init}} policy configured to fallback onto. diff --git a/manage-data/lifecycle/data-stream/tutorial-update-existing-data-stream.md b/manage-data/lifecycle/data-stream/tutorial-update-existing-data-stream.md index 50339b8dbc..e029b38f5b 100644 --- a/manage-data/lifecycle/data-stream/tutorial-update-existing-data-stream.md +++ b/manage-data/lifecycle/data-stream/tutorial-update-existing-data-stream.md @@ -77,8 +77,6 @@ The response will look like: } } ``` -% TEST[continued] -% TESTRESPONSE[skip:the result is for illustrating purposes only] 1. The name of the backing index. 2. This index is managed by a data stream lifecycle. @@ -90,7 +88,8 @@ The response will look like: 8. The time that has passed since this index was [rolled over](../index-lifecycle-management/rollover.md). 9. The time that will be used to determine when it’s safe to delete this index and all its data. 10. The data retention for this index as well is at least 30 days, as it was recently updated. - +% TEST[continued] +% TESTRESPONSE[skip:the result is for illustrating purposes only] ## Remove lifecycle for a data stream [delete-lifecycle] @@ -124,11 +123,10 @@ GET .ds-my-data-stream-*/_lifecycle/explain } } ``` -% TESTRESPONSE[skip:the result is for illustrating purposes only] 1. The name of the backing index. 2. Indication that the index is not managed by the data stream lifecycle. 3. The name of another backing index. 4. Indication that the index is not managed by the data stream lifecycle. - +% TESTRESPONSE[skip:the result is for illustrating purposes only] diff --git a/manage-data/lifecycle/index-lifecycle-management/configure-lifecycle-policy.md b/manage-data/lifecycle/index-lifecycle-management/configure-lifecycle-policy.md index 48025b82b9..ed82806626 100644 --- a/manage-data/lifecycle/index-lifecycle-management/configure-lifecycle-policy.md +++ b/manage-data/lifecycle/index-lifecycle-management/configure-lifecycle-policy.md @@ -188,10 +188,9 @@ PUT mylogs-pre-ilm*/_settings <1> } } ``` -% TEST[continued] 1. Updates all indices with names that start with `mylogs-pre-ilm` - +% TEST[continued] ### Switch lifecycle policies [switch-lifecycle-policies] diff --git a/manage-data/lifecycle/index-lifecycle-management/tutorial-automate-rollover.md b/manage-data/lifecycle/index-lifecycle-management/tutorial-automate-rollover.md index db4395d8f8..43170a0bc3 100644 --- a/manage-data/lifecycle/index-lifecycle-management/tutorial-automate-rollover.md +++ b/manage-data/lifecycle/index-lifecycle-management/tutorial-automate-rollover.md @@ -118,11 +118,10 @@ PUT _index_template/timeseries_template } } ``` -% TEST[continued] 1. Apply the template when a document is indexed into the `timeseries` target. 2. The name of the {{ilm-init}} policy used to manage the data stream. - +% TEST[continued] :::: @@ -203,14 +202,13 @@ The following response shows the data stream’s first generation backing index } } ``` -% TESTRESPONSE[skip:no way to know if we will get this response immediately] 1. The age of the index used for calculating when to rollover the index via the `max_age` 2. The policy used to manage the index 3. The age of the indexed used to transition to the next phase (in this case it is the same with the age of the index). 4. The step {{ilm-init}} is performing on the index 5. The definition of the current phase (the `hot` phase) - +% TESTRESPONSE[skip:no way to know if we will get this response immediately] ## Manage time series data without data streams [manage-time-series-data-without-data-streams] @@ -258,12 +256,11 @@ PUT _index_template/timeseries_template } } ``` -% TEST[continued] 1. Apply the template to a new index if its name starts with `timeseries-`. 2. The name of the lifecycle policy to apply to each new index. 3. The name of the alias used to reference these indices. Required for policies that use the rollover action. - +% TEST[continued] ### Bootstrap the initial time series index with a write index alias [ilm-gs-alias-bootstrap] diff --git a/solutions/search/cross-cluster-search.md b/solutions/search/cross-cluster-search.md index 93541103cc..264d93ffd7 100644 --- a/solutions/search/cross-cluster-search.md +++ b/solutions/search/cross-cluster-search.md @@ -82,11 +82,10 @@ PUT _cluster/settings } } ``` -% TEST[setup:host] -% TEST[s/35.238.149.\d+:930\d+/${transport_host}/] 1. Since `skip_unavailable` was not set on `cluster_three`, it uses the default of `true`. See the [Optional remote clusters](#skip-unavailable-clusters) section for details. - +% TEST[setup:host] +% TEST[s/35.238.149.\d+:930\d+/${transport_host}/] ### Search a single remote cluster [ccs-search-remote-cluster] @@ -172,13 +171,6 @@ The API returns the following response. Note that when you search one or more re } } ``` -% TESTRESPONSE[s/"took": 150/"took": "$body.took"/] -% TESTRESPONSE[s/"max_score": 1/"max_score": "$body.hits.max_score"/] -% TESTRESPONSE[s/"_score": 1/"_score": "$body.hits.hits.0._score"/] -% TESTRESPONSE[s/"total": 12/"total": "$body._shards.total"/] -% TESTRESPONSE[s/"successful": 12/"successful": "$body._shards.successful"/] -% TESTRESPONSE[s/"skipped": 0/"skipped": "$body._shards.skipped"/] -% TESTRESPONSE[s/"took": 148/"took": "$body._clusters.details.cluster_one.took"/] 1. This section of counters shows all possible cluster search states and how many cluster searches are currently in that state. The clusters can be one of the following statuses: **running**, **successful** (searches on all shards were successful), **partial** (searches on at least one shard of the cluster was successful and at least one failed), **skipped** (the search failed on a cluster marked with `skip_unavailable`=`true`) or **failed** (the search failed on a cluster marked with `skip_unavailable`=`false`). 2. The `_clusters/details` section shows metadata about the search on each cluster. @@ -186,7 +178,13 @@ The API returns the following response. Note that when you search one or more re 4. How long (in milliseconds) the sub-search took on that cluster. 5. The shard details for the sub-search on that cluster. 6. The search response body includes the name of the remote cluster in the `_index` parameter. - +% TESTRESPONSE[s/"took": 150/"took": "$body.took"/] +% TESTRESPONSE[s/"max_score": 1/"max_score": "$body.hits.max_score"/] +% TESTRESPONSE[s/"_score": 1/"_score": "$body.hits.hits.0._score"/] +% TESTRESPONSE[s/"total": 12/"total": "$body._shards.total"/] +% TESTRESPONSE[s/"successful": 12/"successful": "$body._shards.successful"/] +% TESTRESPONSE[s/"skipped": 0/"skipped": "$body._shards.skipped"/] +% TESTRESPONSE[s/"took": 148/"took": "$body._clusters.details.cluster_one.took"/] ### Search multiple remote clusters [ccs-search-multi-remote-cluster] @@ -330,6 +328,11 @@ The API returns the following response: } } ``` + +1. The local (querying) cluster is identified as "(local)". +2. This document’s `_index` parameter doesn’t include a cluster name. This means the document came from the local cluster. +3. This document came from `cluster_one`. +4. This document came from `cluster_two`. % TESTRESPONSE[s/"took": 150/"took": "$body.took"/] % TESTRESPONSE[s/"max_score": 1/"max_score": "$body.hits.max_score"/] % TESTRESPONSE[s/"_score": 1/"_score": "$body.hits.hits.0._score"/] @@ -346,12 +349,6 @@ The API returns the following response: % TESTRESPONSE[s/"successful" : 6/"successful": "$body._clusters.details.cluster_two._shards.successful"/] % TESTRESPONSE[s/"took": 141/"took": "$body._clusters.details.cluster_two.took"/] -1. The local (querying) cluster is identified as "(local)". -2. This document’s `_index` parameter doesn’t include a cluster name. This means the document came from the local cluster. -3. This document came from `cluster_one`. -4. This document came from `cluster_two`. - - ## Using async search for {{ccs}} with ccs_minimize_roundtrips=true [ccs-async-search-minimize-roundtrips-true] @@ -428,12 +425,11 @@ The API returns the following response: } } ``` -% TEST[skip: hard to reproduce initial state] 1. The async search id. 2. When `ccs_minimize_roundtrips` = `true` and searches on the remote clusters are still running, this section indicates the number of shards in scope for the local cluster only and any clusters that have finished their search so far. This will be updated to include the total number of shards across all clusters only when the search is completed. When `ccs_minimize_roundtrips`= `false`, the total shard count across all clusters is known up front and will be correct. 3. The `_clusters` section indicates that 3 clusters are in scope for the search and all are currently in the "running" state. - +% TEST[skip: hard to reproduce initial state] If you query the [get async search](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-async-search-submit) endpoint while the query is still running, you will see an update in the `_clusters` and `_shards` section of the response as each cluster finishes its search. @@ -605,6 +601,10 @@ Response: } } ``` + +1. Once the search has finished, the completion_time is present. +2. The `_shards` section is now updated to show that 28 total shards were searched across all clusters and that all were successful. +3. The `_clusters` section shows that searches on all 3 clusters were successful. % TESTRESPONSE[s/FklQYndoTDJ2VEFlMEVBTzFJMGhJVFEaLVlKYndBWWZSMUdicUc4WVlEaFl4ZzoxNTU=/$body.id/] % TESTRESPONSE[s/"is_partial": true/"is_partial": $body.is_partial/] % TESTRESPONSE[s/"is_running": true/"is_running": $body.is_running/] @@ -625,11 +625,6 @@ Response: % TESTRESPONSE[s/"total": \d+/"total": $body.$_path/] % TESTRESPONSE[s/"successful": \d+/"successful": $body.$_path/] -1. Once the search has finished, the completion_time is present. -2. The `_shards` section is now updated to show that 28 total shards were searched across all clusters and that all were successful. -3. The `_clusters` section shows that searches on all 3 clusters were successful. - - ## {{ccs-cap}} failures [cross-cluster-search-failures] @@ -758,7 +753,6 @@ Response: } } ``` -% TEST[skip: hard to reproduce failure results] 1. The search results are marked as partial, since at least one shard search failed. 2. The `_shards` section includes shard failure info. @@ -766,7 +760,7 @@ Response: 4. The `partial` status has been applied to the cluster with partial results. 5. The failed shard count is shown. 6. The shard failures are listed under the cluster/details entry also. - +% TEST[skip: hard to reproduce failure results] Here is an example where both `cluster_one` and `cluster_two` lost connectivity during a {{ccs}}. Since `cluster_one` is marked as `skip_unavailable`=`true`, its status is `skipped` and since `cluster_two` is marked as `skip_unavailable`=`false`, its status is `failed`. Since there was a `failed` cluster, a top level `error` is also present and this returns an HTTP status of 500 (not shown). @@ -862,14 +856,13 @@ Response: } } ``` -% TEST[skip: hard to reproduce failure results] 1. The shard accounting will often be only partial when errors like this occur, since we need to be able to get shard info from remote clusters on each search. 2. `cluster_one` disconnected during the search and it returned no results. Since it is marked in the remote cluster configuration as `skip_unavailable`=`true`, its status is "skipped", which will not fail the entire search. 3. The failures list shows that the remote cluster node disconnected from the querying cluster. 4. `cluster_two` status is "failed", since it is marked in the remote cluster configuration as `skip_unavailable`=`false`. 5. A top level `error` entry is included when there is a "failed" cluster. - +% TEST[skip: hard to reproduce failure results] ## Excluding clusters or indices from a {{ccs}} [exclude-problematic-clusters] @@ -895,11 +888,10 @@ POST /my-index-000001,cluster*:my-index-000001,-cluster_three:*/_async_search < "_source": ["user.id", "message", "http.response.status_code"] } ``` -% TEST[continued] -% TEST[s/ccs_minimize_roundtrips=true/ccs_minimize_roundtrips=true&wait_for_completion_timeout=100ms&keep_on_completion=true/] 1. The `cluster*` notation would naturally include `cluster_one`, `cluster_two` and `cluster_three`. To exclude `cluster_three` use a `-` before the cluster name along with a simple wildcard `*` in the index position. This indicates that you do not want the search to make any contact with `cluster_three`. - +% TEST[continued] +% TEST[s/ccs_minimize_roundtrips=true/ccs_minimize_roundtrips=true&wait_for_completion_timeout=100ms&keep_on_completion=true/] **Exclude a remote index** @@ -916,11 +908,10 @@ POST /my-index-000001,cluster*:my-index-*,cluster_three:-my-index-000001/_async_ "_source": ["user.id", "message", "http.response.status_code"] } ``` -% TEST[continued] -% TEST[s/ccs_minimize_roundtrips=true/ccs_minimize_roundtrips=true&wait_for_completion_timeout=100ms&keep_on_completion=true/] 1. This will **not** exclude `cluster_three` from the search. It will still be contacted and told to search any indexes matching `my-index-*` except for `my-index-000001`. - +% TEST[continued] +% TEST[s/ccs_minimize_roundtrips=true/ccs_minimize_roundtrips=true&wait_for_completion_timeout=100ms&keep_on_completion=true/] ## Using async search for {{ccs}} with ccs_minimize_roundtrips=false [ccs-async-search-minimize-roundtrips-false] @@ -1021,12 +1012,11 @@ The API returns the following response if the query takes longer than the `wait_ } } ``` -% TEST[skip: hard to reproduce intermediate results] 1. All shards from all clusters in scope for the search are listed here. Watch this section and/or the _clusters section for updates to monitor search progress. 2. From the `_clusters` section we can see that all the clusters are in "running" state. 3. The `_clusters` section shows that shard information was successfully gathered from all 3 clusters and the total shard count on each cluster is listed. - +% TEST[skip: hard to reproduce intermediate results] ## Optional remote clusters [skip-unavailable-clusters] diff --git a/solutions/search/full-text/search-relevance/static-scoring-signals.md b/solutions/search/full-text/search-relevance/static-scoring-signals.md index fc30aef81c..3725a330b5 100644 --- a/solutions/search/full-text/search-relevance/static-scoring-signals.md +++ b/solutions/search/full-text/search-relevance/static-scoring-signals.md @@ -34,10 +34,9 @@ GET index/_search } } ``` -% TEST[continued] 1. `pagerank` must be mapped as a [Numeric](elasticsearch://reference/elasticsearch/mapping-reference/number.md) - +% TEST[continued] while with the [`rank_feature` query](elasticsearch://reference/query-languages/query-dsl-rank-feature-query.md) it would look like below: diff --git a/solutions/search/hybrid-semantic-text.md b/solutions/search/hybrid-semantic-text.md index b3d7cbc180..2b52b15414 100644 --- a/solutions/search/hybrid-semantic-text.md +++ b/solutions/search/hybrid-semantic-text.md @@ -81,10 +81,9 @@ POST _reindex?wait_for_completion=false } } ``` -% TEST[skip:TBD] 1. The default batch size for reindexing is 1000. Reducing size to a smaller number makes the update of the reindexing process quicker which enables you to follow the progress closely and detect errors early. - +% TEST[skip:TBD] The call returns a task ID to monitor the progress: diff --git a/solutions/search/querydsl-full-text-filter-tutorial.md b/solutions/search/querydsl-full-text-filter-tutorial.md index 569351722d..988924003b 100644 --- a/solutions/search/querydsl-full-text-filter-tutorial.md +++ b/solutions/search/querydsl-full-text-filter-tutorial.md @@ -101,12 +101,11 @@ PUT /cooking_blog/_mapping } } ``` -% TEST 1. The `standard` analyzer is used by default for `text` fields if an `analyzer` isn’t specified. It’s included here for demonstration purposes. 2. [Multi-fields](elasticsearch://reference/elasticsearch/mapping-reference/multi-fields.md) are used here to index `text` fields as both `text` and `keyword` [data types](elasticsearch://reference/elasticsearch/mapping-reference/field-data-types.md). This enables both full-text search and exact matching/filtering on the same field. Note that if you used [dynamic mapping](../../manage-data/data-store/mapping/dynamic-field-mapping.md), these multi-fields would be created automatically. 3. The [`ignore_above` parameter](elasticsearch://reference/elasticsearch/mapping-reference/ignore-above.md) prevents indexing values longer than 256 characters in the `keyword` field. Again this is the default value, but it’s included here for demonstration purposes. It helps to save disk space and avoid potential issues with Lucene’s term byte-length limit. - +% TEST ::::{tip} Full-text search is powered by [text analysis](full-text/text-analysis-during-search.md). Text analysis normalizes and standardizes text data so it can be efficiently stored in an inverted index and searched in near real-time. Analysis happens at both [index and search time](../../manage-data/data-store/text-analysis/index-search-analysis.md). This tutorial won’t cover analysis in detail, but it’s important to understand how text is processed to create effective search queries. @@ -158,10 +157,9 @@ GET /cooking_blog/_search } } ``` -% TEST[continued] 1. By default, the `match` query uses `OR` logic between the resulting tokens. This means it will match documents that contain either "fluffy" or "pancakes", or both, in the description field. - +% TEST[continued] At search time, {{es}} defaults to the analyzer defined in the field mapping. In this example, we’re using the `standard` analyzer. Using a different analyzer at search time is an [advanced use case](../../manage-data/data-store/text-analysis/index-search-analysis.md#different-analyzers). @@ -205,19 +203,18 @@ At search time, {{es}} defaults to the analyzer defined in the field mapping. In } } ``` -% TESTRESPONSE[s/"took": 0/"took": "$body.took"/] -% TESTRESPONSE[s/"total": 1/"total": $body._shards.total/] -% TESTRESPONSE[s/"successful": 1/"successful": $body._shards.successful/] -% TESTRESPONSE[s/"value": 1/"value": $body.hits.total.value/] -% TESTRESPONSE[s/"max_score": 1.8378843/"max_score": $body.hits.max_score/] -% TESTRESPONSE[s/"_score": 1.8378843/"_score": $body.hits.hits.0._score/] 1. The `hits` object contains the total number of matching documents and their relation to the total. 2. `max_score` is the highest relevance score among all matching documents. In this example, we only have one matching document. 3. `_score` is the relevance score for a specific document, indicating how well it matches the query. Higher scores indicate better matches. In this example the `max_score` is the same as the `_score`, as there is only one matching document. 4. The title contains both "Fluffy" and "Pancakes", matching our search terms exactly. 5. The description includes "fluffiest" and "pancakes", further contributing to the document’s relevance due to the analysis process. - +% TESTRESPONSE[s/"took": 0/"took": "$body.took"/] +% TESTRESPONSE[s/"total": 1/"total": $body._shards.total/] +% TESTRESPONSE[s/"successful": 1/"successful": $body._shards.successful/] +% TESTRESPONSE[s/"value": 1/"value": $body.hits.total.value/] +% TESTRESPONSE[s/"max_score": 1.8378843/"max_score": $body.hits.max_score/] +% TESTRESPONSE[s/"_score": 1.8378843/"_score": $body.hits.hits.0._score/] :::: @@ -325,12 +322,11 @@ GET /cooking_blog/_search } } ``` -% TEST[continued] 1. The `^` syntax applies a boost to specific fields:* `title^3`: The title field is 3 times more important than an unboosted field * `description^2`: The description is 2 times more important * `tags`: No boost applied (equivalent to `^1`) - +% TEST[continued] These boosts help tune relevance, prioritizing matches in the title over the description, and matches in the description over tags. @@ -379,14 +375,13 @@ Learn more about fields and per-field boosting in the [`multi_match` query](elas } } ``` -% TESTRESPONSE[s/"took": 0/"took": "$body.took"/] -% TESTRESPONSE[s/"_score": 7.546015/"_score": $body.hits.hits.0._score/] -% TESTRESPONSE[s/"max_score": 7.546015/"max_score": $body.hits.max_score/] 1. The title contains "Vegetarian" and "Curry", which matches our search terms. The title field has the highest boost (^3), contributing significantly to this document’s relevance score. 2. The description contains "curry" and related terms like "vegetables", further increasing the document’s relevance. 3. The tags include both "vegetarian" and "curry", providing an exact match for our search terms, albeit with no boost. - +% TESTRESPONSE[s/"took": 0/"took": "$body.took"/] +% TESTRESPONSE[s/"_score": 7.546015/"_score": $body.hits.hits.0._score/] +% TESTRESPONSE[s/"max_score": 7.546015/"max_score": $body.hits.max_score/] This result demonstrates how the `multi_match` query with field boosts helps users find relevant recipes across multiple fields. Even though the exact phrase "vegetarian curry" doesn’t appear in any single field, the combination of matches across fields produces a highly relevant result. @@ -418,10 +413,9 @@ GET /cooking_blog/_search } } ``` -% TEST[continued] 1. Note the use of `category.keyword` here. This refers to the [`keyword`](elasticsearch://reference/elasticsearch/mapping-reference/keyword.md) multi-field of the `category` field, ensuring an exact, case-sensitive match. - +% TEST[continued] ::::{tip} The `.keyword` suffix accesses the unanalyzed version of a field, enabling exact, case-sensitive matching. This works in two scenarios: @@ -450,11 +444,10 @@ GET /cooking_blog/_search } } ``` -% TEST[continued] 1. Greater than or equal to May 1, 2023. 2. Less than or equal to May 31, 2023. - +% TEST[continued] ### Find exact matches [full-text-filter-tutorial-term-query] @@ -473,10 +466,9 @@ GET /cooking_blog/_search } } ``` -% TEST[continued] 1. The `term` query has zero flexibility. For example, here the queries `maria` or `maria rodriguez` would have zero hits, due to case sensitivity. - +% TEST[continued] ::::{tip} Avoid using the `term` query for [`text` fields](elasticsearch://reference/elasticsearch/mapping-reference/text.md) because they are transformed by the analysis process. @@ -547,10 +539,9 @@ GET /cooking_blog/_search } } ``` -% TEST[continued] 1. The `must_not` clause excludes documents that match the specified criteria. This is a powerful tool for filtering out unwanted results. - +% TEST[continued] ::::{dropdown} Example response ```console-result @@ -593,7 +584,6 @@ GET /cooking_blog/_search } } ``` -% TESTRESPONSE[s/"took": 1/"took": "$body.took"/] 1. The title contains "Spicy" and "Curry", matching our should condition. With the default [best_fields](elasticsearch://reference/query-languages/query-dsl-multi-match-query.md#type-best-fields) behavior, this field contributes most to the relevance score. 2. While the description also contains matching terms, only the best matching field’s score is used by default. @@ -601,7 +591,7 @@ GET /cooking_blog/_search 4. The "Main Course" category satisfies another `should` condition. 5. The "vegetarian" tag satisfies a `must` condition, while "curry" and "spicy" tags align with our `should` preferences. 6. The rating of 4.6 meets our minimum rating requirement of 4.5. - +% TESTRESPONSE[s/"took": 1/"took": "$body.took"/] :::: diff --git a/solutions/search/ranking/learning-to-rank-search-usage.md b/solutions/search/ranking/learning-to-rank-search-usage.md index 65cc4fd5ce..fb737a7f0e 100644 --- a/solutions/search/ranking/learning-to-rank-search-usage.md +++ b/solutions/search/ranking/learning-to-rank-search-usage.md @@ -42,13 +42,12 @@ GET my-index/_search } } ``` -% TEST[skip:TBD] 1. First pass query providing documents to be rescored. 2. The unique identifier of the trained model uploaded to {{es}}. 3. Named parameters to be passed to the query templates used for feature. 4. The number of documents that should be examined by the rescorer on each shard. - +% TEST[skip:TBD] ### Known limitations [learning-to-rank-rescorer-limitations] diff --git a/solutions/search/search-applications/search-application-security.md b/solutions/search/search-applications/search-application-security.md index d24e15bd1f..f351f131d6 100644 --- a/solutions/search/search-applications/search-application-security.md +++ b/solutions/search/search-applications/search-application-security.md @@ -58,11 +58,10 @@ POST /_security/api_key } } ``` -% TEST[skip:TODO] 1. `indices.name` must be the name(s) of the Search Application(s), not the underlying {{es}} indices. 2. `restriction.workflows` must be set to the concrete value `search_application_query`. - +% TEST[skip:TODO] ::::{important} It is crucial to specify the workflow restriction. Without this the {{es}} API key can directly call `_search` and issue arbitrary {{es}} queries. This is insecure when dealing with untrusted clients. diff --git a/solutions/search/semantic-search/semantic-search-elser-ingest-pipelines.md b/solutions/search/semantic-search/semantic-search-elser-ingest-pipelines.md index a7a29ea1a5..9e095e2dcc 100644 --- a/solutions/search/semantic-search/semantic-search-elser-ingest-pipelines.md +++ b/solutions/search/semantic-search/semantic-search-elser-ingest-pipelines.md @@ -59,13 +59,12 @@ PUT my-index } } ``` -% TEST[skip:TBD] 1. The name of the field to contain the generated tokens. It must be referenced in the {{infer}} pipeline configuration in the next step. 2. The field to contain the tokens is a `sparse_vector` field. 3. The name of the field from which to create the sparse vector representation. In this example, the name of the field is `content`. It must be referenced in the {{infer}} pipeline configuration in the next step. 4. The field type which is text in this example. - +% TEST[skip:TBD] To learn how to optimize space, refer to the [Saving disk space by excluding the ELSER tokens from document source](/manage-data/ingest/transform-enrich/ingest-pipelines.md) with an [{{infer}} processor](elasticsearch://reference/ingestion-tools/enrich-processor/inference-processor.md) to use ELSER to infer against the data that is being ingested in the pipeline. @@ -123,10 +122,9 @@ POST _reindex?wait_for_completion=false } } ``` -% TEST[skip:TBD] 1. The default batch size for reindexing is 1000. Reducing `size` to a smaller number makes the update of the reindexing process quicker which enables you to follow the progress closely and detect errors early. - +% TEST[skip:TBD] The call returns a task ID to monitor the progress: @@ -236,13 +234,12 @@ GET my-index/_search "min_score": 10 <4> } ``` -% TEST[skip:TBD] 1. Both the `sparse_vector` and the `query_string` queries are in a `should` clause of a `bool` query. 2. The `boost` value is `1` for the `sparse_vector` query which is the default value. This means that the relevance score of the results of this query are not boosted. 3. The `boost` value is `4` for the `query_string` query. The relevance score of the results of this query is increased causing them to rank higher in the search results. 4. Only the results with a score equal to or higher than `10` are displayed. - +% TEST[skip:TBD] ## Optimizing performance [optimization] diff --git a/solutions/search/semantic-search/semantic-search-semantic-text.md b/solutions/search/semantic-search/semantic-search-semantic-text.md index f265603e95..b7ae005705 100644 --- a/solutions/search/semantic-search/semantic-search-semantic-text.md +++ b/solutions/search/semantic-search/semantic-search-semantic-text.md @@ -45,11 +45,10 @@ PUT semantic-embeddings } } ``` -% TEST[skip:TBD] 1. The name of the field to contain the generated embeddings. 2. The field to contain the embeddings is a `semantic_text` field. Since no `inference_id` is provided, the default endpoint `.elser-2-elasticsearch` for the [`elasticsearch` service](../../../explore-analyze/elastic-inference/inference-api/elasticsearch-inference-integration.md) is used. To use a different {{infer}} service, you must create an {{infer}} endpoint first using the [Create {{infer}} API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put) and then specify it in the `semantic_text` field mapping using the `inference_id` parameter. - +% TEST[skip:TBD] ::::{note} If you’re using web crawlers or connectors to generate indices, you have to [update the index mappings](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-put-mapping) for these indices to include the `semantic_text` field. Once the mapping is updated, you’ll need to run a full web crawl or a full connector sync. This ensures that all existing documents are reprocessed and updated with the new semantic embeddings, enabling semantic search on the updated data. @@ -89,10 +88,9 @@ POST _reindex?wait_for_completion=false } } ``` -% TEST[skip:TBD] 1. The default batch size for reindexing is 1000. Reducing size to a smaller number makes the update of the reindexing process quicker which enables you to follow the progress closely and detect errors early. - +% TEST[skip:TBD] The call returns a task ID to monitor the progress: @@ -124,11 +122,10 @@ GET semantic-embeddings/_search } } ``` -% TEST[skip:TBD] 1. The `semantic_text` field on which you want to perform the search. 2. The query text. - +% TEST[skip:TBD] As a result, you receive the top 10 documents that are closest in meaning to the query from the `semantic-embedding` index. diff --git a/solutions/search/vector/bring-own-vectors.md b/solutions/search/vector/bring-own-vectors.md index aa50656081..df9cc60304 100644 --- a/solutions/search/vector/bring-own-vectors.md +++ b/solutions/search/vector/bring-own-vectors.md @@ -54,12 +54,11 @@ PUT /amazon-reviews } } ``` -% TEST SETUP 1. The `dims` parameter must match the length of the embedding vector. Here we’re using a simple 8-dimensional embedding for readability. If not specified, `dims` will be dynamically calculated based on the first indexed document. 2. The `index` parameter is set to `true` to enable the use of the `knn` query. 3. The `similarity` parameter defines the similarity function used to compare the query vector to the document vectors. `cosine` is the default similarity function for `dense_vector` fields in {{es}}. - +% TEST SETUP ## Step 2: Index documents with embeddings [bring-your-own-vectors-index-documents] @@ -76,10 +75,9 @@ PUT /amazon-reviews/_doc/1 "review_vector": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8] <1> } ``` -% TEST 1. The size of the `review_vector` array is 8, matching the `dims` count specified in the mapping. - +% TEST ### Bulk index multiple documents [_bulk_index_multiple_documents] @@ -119,12 +117,11 @@ POST /amazon-reviews/_search } } ``` -% TEST[skip:flakeyknnerror] 1. In this simple example, we’re sending a raw vector as the query text. In a real-world scenario, you’ll need to generate vectors for queries using an embedding model. 2. The `k` parameter specifies the number of results to return. 3. The `num_candidates` parameter is optional. It limits the number of candidates returned by the search node. This can improve performance and reduce costs. - +% TEST[skip:flakeyknnerror] ## Learn more [bring-your-own-vectors-learn-more] diff --git a/solutions/search/vector/dense-versus-sparse-ingest-pipelines.md b/solutions/search/vector/dense-versus-sparse-ingest-pipelines.md index 0e3dc537bf..1ddade7fb9 100644 --- a/solutions/search/vector/dense-versus-sparse-ingest-pipelines.md +++ b/solutions/search/vector/dense-versus-sparse-ingest-pipelines.md @@ -200,7 +200,7 @@ GET my-index/_search } ``` % TEST[skip:TBD] -:::::: + :::::: ::::::{tab-item} Dense vector models @@ -223,7 +223,7 @@ GET my-index/_search } ``` % TEST[skip:TBD] -:::::: + :::::: ::::::: @@ -271,7 +271,7 @@ GET my-index/_search } ``` % TEST[skip:TBD] -:::::: + :::::: ::::::{tab-item} Dense vector models @@ -315,7 +315,7 @@ GET my-index/_search } ``` % TEST[skip:TBD] -:::::: + :::::: ::::::: diff --git a/solutions/search/vector/knn.md b/solutions/search/vector/knn.md index a20d2a50fe..458c7f1a1b 100644 --- a/solutions/search/vector/knn.md +++ b/solutions/search/vector/knn.md @@ -263,9 +263,9 @@ PUT quantized-image-index } } ``` -% TEST[continued] 1. Index your `float` vectors. +% TEST[continued] ```console POST quantized-image-index/_bulk?refresh=true @@ -449,12 +449,11 @@ Reference the deployed text embedding model or the model deployment in the `quer } (...) ``` -% NOTCONSOLE 1. The {{nlp}} task to perform. It must be `text_embedding`. 2. The ID of the text embedding model to use to generate the dense vectors from the query string. Use the same model that generated the embeddings from the input text in the index you search against. You can use the value of the `deployment_id` instead in the `model_id` argument. 3. The query string from which the model generates the dense vector representation. - +% NOTCONSOLE For more information on how to deploy a trained model and use it to create text embeddings, refer to this [end-to-end example](../../../explore-analyze/machine-learning/nlp/ml-nlp-text-emb-vector-search-example.md). @@ -1015,7 +1014,6 @@ POST /my-index/_search } } ``` -% TEST[skip: setup not provided] 1. The number of results to return, note its only 10 and we will oversample by 2x, gathering 20 nearest neighbors. 2. The number of results to return from the KNN search. This will do an approximate KNN search with 50 candidates per HNSW graph and use the quantized vectors, returning the 20 most similar vectors according to the quantized score. Additionally, since this is the top-level `knn` object, the global top 20 results will from all shards will be gathered before rescoring. Combining with `rescore`, this is oversampling by `2x`, meaning gathering 20 nearest neighbors according to quantized scoring and rescoring with higher fidelity float vectors. @@ -1023,7 +1021,7 @@ POST /my-index/_search 4. The script to rescore the results. Script score will interact directly with the originally provided float32 vector. 5. The weight of the original query, here we simply throw away the original score 6. The weight of the rescore query, here we only use the rescore query - +% TEST[skip: setup not provided] ##### Use a `script_score` query to rescore per shard [dense-vector-knn-search-rescoring-script-score] @@ -1055,13 +1053,12 @@ POST /my-index/_search } } ``` -% TEST[skip: setup not provided] 1. The number of results to return 2. The `knn` query to perform the initial search, this is executed per-shard 3. The number of candidates to use for the initial approximate `knn` search. This will search using the quantized vectors and return the top 20 candidates per shard to then be scored 4. The script to score the results. Script score will interact directly with the originally provided float32 vector. - +% TEST[skip: setup not provided] ## Exact kNN [exact-knn] diff --git a/troubleshoot/elasticsearch/add-tier.md b/troubleshoot/elasticsearch/add-tier.md index bbfcd7a66e..5a2f5f1749 100644 --- a/troubleshoot/elasticsearch/add-tier.md +++ b/troubleshoot/elasticsearch/add-tier.md @@ -83,9 +83,10 @@ The response will look like this: } } ``` -% TESTRESPONSE[skip:the result is for illustrating purposes only] 1. Represents a comma-separated list of data tier node roles this index is allowed to be allocated on, the first one in the list being the one with the higher priority i.e. the tier the index is targeting. e.g. in this example the tier preference is `data_warm,data_hot` so the index is targeting the `warm` tier and more nodes with the `data_warm` role are needed in the {{es}} cluster. +% TESTRESPONSE[skip:the result is for illustrating purposes only] + :::::: ::::::: diff --git a/troubleshoot/elasticsearch/increase-shard-limit.md b/troubleshoot/elasticsearch/increase-shard-limit.md index 489efd7d55..e0cd749acd 100644 --- a/troubleshoot/elasticsearch/increase-shard-limit.md +++ b/troubleshoot/elasticsearch/increase-shard-limit.md @@ -90,10 +90,9 @@ The response will look like this: } } ``` -% TESTRESPONSE[skip:the result is for illustrating purposes only] 1. Represents a comma separated list of data tier node roles this index is allowed to be allocated on, the first one in the list being the one with the higher priority i.e. the tier the index is targeting. e.g. in this example the tier preference is `data_warm,data_hot` so the index is targeting the `warm` tier and more nodes with the `data_warm` role are needed in the {{es}} cluster. - +% TESTRESPONSE[skip:the result is for illustrating purposes only] Alternatively, if adding more nodes to the {{es}} cluster is not desired, inspecting the configuration for the `index.routing.allocation.total_shards_per_node` [index setting](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-get-settings) and increasing the configured value will allow more shards to be assigned on the same node. @@ -129,6 +128,6 @@ Alternatively, if adding more nodes to the {{es}} cluster is not desired, inspec ``` :::::: -:::{tip}} +:::{tip} If you’re using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to [Monitor with AutoOps](/deploy-manage/monitor/autoops.md). ::: diff --git a/troubleshoot/elasticsearch/increase-tier-capacity.md b/troubleshoot/elasticsearch/increase-tier-capacity.md index f19c99189d..7a0c510816 100644 --- a/troubleshoot/elasticsearch/increase-tier-capacity.md +++ b/troubleshoot/elasticsearch/increase-tier-capacity.md @@ -52,10 +52,9 @@ The response will look like this: } } ``` -% TESTRESPONSE[skip:the result is for illustrating purposes only] 1. Represents a comma separated list of data tier node roles this index is allowed to be allocated on, the first one in the list being the one with the higher priority i.e. the tier the index is targeting. e.g. in this example the tier preference is `data_warm,data_hot` so the index is targeting the `warm` tier and more nodes with the `data_warm` role are needed in the {{es}} cluster. - +% TESTRESPONSE[skip:the result is for illustrating purposes only] Now that you know the tier, you want to increase the number of nodes in that tier so that the replicas can be allocated. To do this you can either increase the size per zone to increase the number of nodes in the availability zone(s) you were already using, or increase the number of availability zones. Go back to the deployment’s landing page by clicking on the three horizontal bars on the top left of the screen and choosing **Manage this deployment**. On that page click the **Manage** button, and choose **Edit deployment**. Note that you must be logged in to [https://cloud.elastic.co/](https://cloud.elastic.co/) in order to do this. In the {{es}} section, find the tier where the replica shards could not be assigned. @@ -149,10 +148,9 @@ The response will look like this: } } ``` -% TESTRESPONSE[skip:the result is for illustrating purposes only] 1. Represents a comma separated list of data tier node roles this index is allowed to be allocated on, the first one in the list being the one with the higher priority i.e. the tier the index is targeting. e.g. in this example the tier preference is `data_warm,data_hot` so the index is targeting the `warm` tier and more nodes with the `data_warm` role are needed in the {{es}} cluster. - +% TESTRESPONSE[skip:the result is for illustrating purposes only] Alternatively, if adding more nodes to the {{es}} cluster is not desired, inspect the [`index.number_of_replicas`](elasticsearch://reference/elasticsearch/index-settings/index-modules.md#dynamic-index-number-of-replicas) index setting and decrease the configured value: diff --git a/troubleshoot/elasticsearch/index-lifecycle-management-errors.md b/troubleshoot/elasticsearch/index-lifecycle-management-errors.md index 746e073ccc..4e86f0c002 100644 --- a/troubleshoot/elasticsearch/index-lifecycle-management-errors.md +++ b/troubleshoot/elasticsearch/index-lifecycle-management-errors.md @@ -97,7 +97,6 @@ Which returns the following information: } } ``` -% TESTRESPONSE[skip:no way to know if we will get this response immediately] 1. The policy being used to manage the index: `shrink-index` 2. The index age: 5.1 days @@ -107,7 +106,7 @@ Which returns the following information: 6. The step that failed to execute: `shrink` 7. The type of error and a description of that error. 8. The definition of the current phase from the `shrink-index` policy - +% TESTRESPONSE[skip:no way to know if we will get this response immediately] To resolve this, you could update the policy to shrink the index to a single shard after 5 days: diff --git a/troubleshoot/elasticsearch/remote-clusters.md b/troubleshoot/elasticsearch/remote-clusters.md index 91b655d6f7..da76ee4394 100644 --- a/troubleshoot/elasticsearch/remote-clusters.md +++ b/troubleshoot/elasticsearch/remote-clusters.md @@ -43,11 +43,10 @@ The API should return `"connected" : true`. When using [API key authentication]( } } ``` -% TEST[skip:TODO] 1. The remote cluster has connected successfully. 2. If present, indicates the remote cluster has connected using [API key authentication](../../deploy-manage/remote-clusters/remote-clusters-api-key.md) instead of [certificate based authentication](../../deploy-manage/remote-clusters/remote-clusters-cert.md). - +% TEST[skip:TODO] ### Enabling the remote cluster server [remote-clusters-troubleshooting-enable-server] @@ -268,11 +267,10 @@ The API should return `"connected" : true`. When using [API key authentication]( } } ``` -% TEST[skip:TODO] 1. The remote cluster has connected successfully. 2. If present, indicates the remote cluster has connected using [API key authentication](../../deploy-manage/remote-clusters/remote-clusters-api-key.md) instead of [certificate based authentication](../../deploy-manage/remote-clusters/remote-clusters-cert.md). - +% TEST[skip:TODO] Besides checking the response of the remote cluster info API, you can also check the logs. diff --git a/troubleshoot/elasticsearch/repeated-snapshot-failures.md b/troubleshoot/elasticsearch/repeated-snapshot-failures.md index c363a6f27e..823306b589 100644 --- a/troubleshoot/elasticsearch/repeated-snapshot-failures.md +++ b/troubleshoot/elasticsearch/repeated-snapshot-failures.md @@ -153,13 +153,12 @@ The response will look like this: } } ``` -% TESTRESPONSE[skip:the result is for illustrating purposes only] 1. The affected snapshot lifecycle policy. 2. The information about the last failure for the policy. 3. The time when the failure occurred in millis. Use the `human=true` request parameter to see a formatted timestamp. 4. Error details containing the reason for the snapshot failure. - +% TESTRESPONSE[skip:the result is for illustrating purposes only] Snapshots can fail for a variety reasons. If the failures are due to configuration errors, consult the documentation for the repository that the automated snapshots are using. diff --git a/troubleshoot/elasticsearch/start-ilm.md b/troubleshoot/elasticsearch/start-ilm.md index 4f4f3956ca..eddd4b57f5 100644 --- a/troubleshoot/elasticsearch/start-ilm.md +++ b/troubleshoot/elasticsearch/start-ilm.md @@ -103,7 +103,7 @@ The response will look like this: } ``` % TESTRESPONSE[skip:the result is for illustrating purposes only] -:::::: + :::::: ::::::: @@ -194,7 +194,7 @@ The response will look like this: } ``` % TESTRESPONSE[skip:the result is for illustrating purposes only] -:::::: + :::::: ::::::: diff --git a/troubleshoot/elasticsearch/troubleshooting-shards-capacity-issues.md b/troubleshoot/elasticsearch/troubleshooting-shards-capacity-issues.md index b6b861fd2b..bcfddcf4ca 100644 --- a/troubleshoot/elasticsearch/troubleshooting-shards-capacity-issues.md +++ b/troubleshoot/elasticsearch/troubleshooting-shards-capacity-issues.md @@ -163,11 +163,10 @@ The response will look like this: } } ``` -% TESTRESPONSE[skip:the result is for illustrating purposes only] 1. Current value of the setting `cluster.max_shards_per_node` 2. Current number of open shards across the cluster - +% TESTRESPONSE[skip:the result is for illustrating purposes only] Using the [`cluster settings API`](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-cluster-put-settings), update the [`cluster.max_shards_per_node`](elasticsearch://reference/elasticsearch/configuration-reference/miscellaneous-cluster-settings.md#cluster-max-shards-per-node) setting: @@ -372,11 +371,10 @@ GET _health_report/shards_capacity } } ``` -% TESTRESPONSE[skip:the result is for illustrating purposes only] 1. Current value of the setting `cluster.max_shards_per_node.frozen`. 2. Current number of open shards used by frozen nodes across the cluster. - +% TESTRESPONSE[skip:the result is for illustrating purposes only] Using the [`cluster settings API`](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-cluster-put-settings), update the [`cluster.max_shards_per_node.frozen`](elasticsearch://reference/elasticsearch/configuration-reference/miscellaneous-cluster-settings.md#cluster-max-shards-per-node-frozen) setting: