From f2601cc4b4ef005d9ac9efffef1d0599bace275a Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Fri, 20 Sep 2024 11:36:47 -0400 Subject: [PATCH 01/10] Deprecate legacy params from range query Deprecate to, from, include_lower, include_upper range query params. These params have been removed from our documentation in v. 0.90.4 (d6ecdecc19d54b4a37b033a2237a4c76601e9a2d), but did not got through deprecation cycle. These params to be removed in v9.0. Related to #81276 Closes #48538 --- .../watcher/transform/search.asciidoc | 8 +++--- .../test/search/330_fetch_fields.yml | 2 +- .../test/search/500_date_range.yml | 26 +++++++++++++++++++ .../index/query/RangeQueryBuilder.java | 8 +++--- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/docs/reference/watcher/transform/search.asciidoc b/docs/reference/watcher/transform/search.asciidoc index deef57783ea91..21db8293f3fa2 100644 --- a/docs/reference/watcher/transform/search.asciidoc +++ b/docs/reference/watcher/transform/search.asciidoc @@ -119,8 +119,8 @@ time of the watch: { "range" : { "@timestamp" : { - "from" : "{{ctx.trigger.scheduled_time}}||-30s", - "to" : "{{ctx.trigger.triggered_time}}" + "gte" : "{{ctx.trigger.scheduled_time}}||-30s", + "lte" : "{{ctx.trigger.triggered_time}}" } } } @@ -159,8 +159,8 @@ The following is an example of using templates that refer to provided parameters { "range" : { "@timestamp" : { - "from" : "{{ctx.trigger.scheduled_time}}||-30s", - "to" : "{{ctx.trigger.triggered_time}}" + "gte" : "{{ctx.trigger.scheduled_time}}||-30s", + "lte" : "{{ctx.trigger.triggered_time}}" } } } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml index c120bed2d369d..8a8dffda69e20 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml @@ -252,7 +252,7 @@ query: range: date: - from: "1990-12-29T22:30:00.000Z" + gte: "1990-12-29T22:30:00.000Z" fields: - field: date format: "yyyy/MM/dd" diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/500_date_range.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/500_date_range.yml index 76057b5a364fb..e9bfffb8da604 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/500_date_range.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/500_date_range.yml @@ -123,3 +123,29 @@ setup: - match: { hits.total: 1 } - length: { hits.hits: 1 } - match: { hits.hits.0._id: "4" } + + +--- +"from, to, include_lower, include_upper deprecated": + - requires: + cluster_features: "gte_v8.16.0" + reason: 'from, to, include_lower, include_upper parameters are deprecated since 8.16.0' + test_runner_features: warnings + + - do: + warnings: + - "Deprecated field [from] used, this field is unused and will be removed entirely" + - "Deprecated field [to] used, this field is unused and will be removed entirely" + - "Deprecated field [include_lower] used, this field is unused and will be removed entirely" + - "Deprecated field [include_upper] used, this field is unused and will be removed entirely" + search: + index: dates + body: + sort: field + query: + range: + date: + from: 1000 + to: 2023 + include_lower: false + include_upper: false diff --git a/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java index 381c6619af0bb..f1081d06d649d 100644 --- a/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java @@ -45,10 +45,10 @@ public class RangeQueryBuilder extends AbstractQueryBuilder i public static final ParseField LTE_FIELD = new ParseField("lte"); public static final ParseField GTE_FIELD = new ParseField("gte"); - public static final ParseField FROM_FIELD = new ParseField("from"); - public static final ParseField TO_FIELD = new ParseField("to"); - private static final ParseField INCLUDE_LOWER_FIELD = new ParseField("include_lower"); - private static final ParseField INCLUDE_UPPER_FIELD = new ParseField("include_upper"); + public static final ParseField FROM_FIELD = new ParseField("from").withAllDeprecated(); + public static final ParseField TO_FIELD = new ParseField("to").withAllDeprecated(); + private static final ParseField INCLUDE_LOWER_FIELD = new ParseField("include_lower").withAllDeprecated(); + private static final ParseField INCLUDE_UPPER_FIELD = new ParseField("include_upper").withAllDeprecated(); public static final ParseField GT_FIELD = new ParseField("gt"); public static final ParseField LT_FIELD = new ParseField("lt"); private static final ParseField TIME_ZONE_FIELD = new ParseField("time_zone"); From 54850e15faaaa10f1c3aa2cf0284cb49c0a1af36 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Fri, 20 Sep 2024 11:45:13 -0400 Subject: [PATCH 02/10] Update docs/changelog/113286.yaml --- docs/changelog/113286.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 docs/changelog/113286.yaml diff --git a/docs/changelog/113286.yaml b/docs/changelog/113286.yaml new file mode 100644 index 0000000000000..3dfc1e0074b9b --- /dev/null +++ b/docs/changelog/113286.yaml @@ -0,0 +1,12 @@ +pr: 113286 +summary: Deprecate legacy params from range query +area: Search +type: deprecation +issues: + - 48538 +deprecation: + title: Deprecate legacy params from range query + area: Search + details: Please describe the details of this change for the release notes. You can + use asciidoc. + impact: Please describe the impact of this change to users From 3386dc6401f663daa674f50f2a8417b76ef9f24e Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Fri, 20 Sep 2024 11:57:58 -0400 Subject: [PATCH 03/10] Update docs/changelog/113286.yaml --- docs/changelog/113286.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/changelog/113286.yaml b/docs/changelog/113286.yaml index 3dfc1e0074b9b..e125cf7dbc96c 100644 --- a/docs/changelog/113286.yaml +++ b/docs/changelog/113286.yaml @@ -2,8 +2,7 @@ pr: 113286 summary: Deprecate legacy params from range query area: Search type: deprecation -issues: - - 48538 +issues: [] deprecation: title: Deprecate legacy params from range query area: Search From 6948107ec54cc003cccd3130e8d0d23bf660937b Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Fri, 20 Sep 2024 13:15:07 -0400 Subject: [PATCH 04/10] Adjust change log --- docs/changelog/113286.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/changelog/113286.yaml b/docs/changelog/113286.yaml index e125cf7dbc96c..eeffb10b4e638 100644 --- a/docs/changelog/113286.yaml +++ b/docs/changelog/113286.yaml @@ -5,7 +5,6 @@ type: deprecation issues: [] deprecation: title: Deprecate legacy params from range query - area: Search - details: Please describe the details of this change for the release notes. You can - use asciidoc. - impact: Please describe the impact of this change to users + area: REST API + details: Range query will not longer accept `to`, `from`, `include_lower`, and `include_upper` parameters. + impact: Instead use `gt`, `gte`, `lt` and `lte` parameters. From 2a357ccef7b1a24cb37b2cd0633642fa0e8224af Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Fri, 20 Sep 2024 14:53:49 -0400 Subject: [PATCH 05/10] More substitutions --- .../bucket/histogram-aggregation.asciidoc | 4 +- .../watcher/getting-started.asciidoc | 2 +- .../test/indices.sort/10_basic.yml | 6 +-- .../index/query/RangeQueryBuilderTests.java | 42 +------------------ 4 files changed, 8 insertions(+), 46 deletions(-) diff --git a/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc index 12f7cbb310d74..c2ae23dccb7dc 100644 --- a/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc @@ -165,7 +165,7 @@ Example: POST /sales/_search?size=0 { "query": { - "constant_score": { "filter": { "range": { "price": { "to": "500" } } } } + "constant_score": { "filter": { "range": { "price": { "lte": "500" } } } } }, "aggs": { "prices": { @@ -202,7 +202,7 @@ Example: POST /sales/_search?size=0 { "query": { - "constant_score": { "filter": { "range": { "price": { "to": "500" } } } } + "constant_score": { "filter": { "range": { "price": { "lte": "500" } } } } }, "aggs": { "prices": { diff --git a/docs/reference/watcher/getting-started.asciidoc b/docs/reference/watcher/getting-started.asciidoc index c5e7c108742ed..630445f995e07 100644 --- a/docs/reference/watcher/getting-started.asciidoc +++ b/docs/reference/watcher/getting-started.asciidoc @@ -135,7 +135,7 @@ GET .watcher-history*/_search?pretty "bool" : { "must" : [ { "match" : { "result.condition.met" : true }}, - { "range" : { "result.execution_time" : { "from" : "now-10s" }}} + { "range" : { "result.execution_time" : { "gte" : "now-10s" }}} ] } } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.sort/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.sort/10_basic.yml index 68508ec9ad45b..625ccc84e4261 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.sort/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.sort/10_basic.yml @@ -101,7 +101,7 @@ index: test body: sort: ["rank"] - query: {"range": { "rank": { "from": 0 } } } + query: {"range": { "rank": { "gte": 0 } } } track_total_hits: false size: 1 @@ -142,7 +142,7 @@ index: test body: sort: ["rank"] - query: {"range": { "rank": { "from": 0 } } } + query: {"range": { "rank": { "gte": 0 } } } track_total_hits: false size: 3 @@ -160,6 +160,6 @@ scroll: 1m body: sort: ["rank"] - query: {"range": { "rank": { "from": 0 } } } + query: {"range": { "rank": { "gte": 0 } } } track_total_hits: false size: 3 diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java index 4b7388df5d2b9..9de02312e7bab 100644 --- a/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java @@ -111,44 +111,6 @@ protected RangeQueryBuilder doCreateTestQueryBuilder() { return query; } - @Override - protected Map getAlternateVersions() { - Map alternateVersions = new HashMap<>(); - RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder(INT_FIELD_NAME); - - rangeQueryBuilder.includeLower(randomBoolean()); - rangeQueryBuilder.includeUpper(randomBoolean()); - - if (randomBoolean()) { - rangeQueryBuilder.from(randomIntBetween(1, 100)); - } - - if (randomBoolean()) { - rangeQueryBuilder.to(randomIntBetween(101, 200)); - } - - String query = Strings.format( - """ - { - "range":{ - "%s": { - "include_lower":%s, - "include_upper":%s, - "from":%s, - "to":%s - } - } - }""", - INT_FIELD_NAME, - rangeQueryBuilder.includeLower(), - rangeQueryBuilder.includeUpper(), - rangeQueryBuilder.from(), - rangeQueryBuilder.to() - ); - alternateVersions.put(query, rangeQueryBuilder); - return alternateVersions; - } - @Override protected void doAssertLuceneQuery(RangeQueryBuilder queryBuilder, Query query, SearchExecutionContext context) throws IOException { String expectedFieldName = expectedFieldName(queryBuilder.fieldName()); @@ -420,8 +382,8 @@ public void testNamedQueryParsing() throws IOException { { "range" : { "timestamp" : { - "from" : "2015-01-01 00:00:00", - "to" : "now", + "gte" : "2015-01-01 00:00:00", + "lte" : "now", "boost" : 1.0, "_name" : "my_range" } From 81d3130183e006b828467c5e02c52bb6dd1b4bf7 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Fri, 20 Sep 2024 14:57:56 -0400 Subject: [PATCH 06/10] Spotless --- .../org/elasticsearch/index/query/RangeQueryBuilderTests.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java index 9de02312e7bab..410a6695ed0ae 100644 --- a/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java @@ -35,8 +35,6 @@ import java.time.Instant; import java.time.ZoneOffset; import java.time.ZonedDateTime; -import java.util.HashMap; -import java.util.Map; import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; import static org.hamcrest.Matchers.equalTo; From 3dc8e92a5b431277849c474113d5d5b1fdf14a06 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Mon, 23 Sep 2024 06:42:10 -0400 Subject: [PATCH 07/10] Test failrues --- .../resources/rest-api-spec/test/rank_eval/30_failures.yml | 2 +- .../rest-api-spec/test/runtime_fields/110_composite.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/rank-eval/src/yamlRestTest/resources/rest-api-spec/test/rank_eval/30_failures.yml b/modules/rank-eval/src/yamlRestTest/resources/rest-api-spec/test/rank_eval/30_failures.yml index c88a769b8687b..2a2b062cb9cdc 100644 --- a/modules/rank-eval/src/yamlRestTest/resources/rest-api-spec/test/rank_eval/30_failures.yml +++ b/modules/rank-eval/src/yamlRestTest/resources/rest-api-spec/test/rank_eval/30_failures.yml @@ -21,7 +21,7 @@ }, { "id" : "invalid_query", - "request": { "query": { "range" : { "bar" : { "from" : "Basel", "time_zone": "+01:00" }}}}, + "request": { "query": { "range" : { "bar" : { "gte" : "Basel", "time_zone": "+01:00" }}}}, "ratings": [{"_index": "foo", "_id": "doc1", "rating": 1}] } ], diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/110_composite.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/110_composite.yml index 13af92f9aa84b..eb6ce4ae61ae2 100644 --- a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/110_composite.yml +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/110_composite.yml @@ -83,8 +83,8 @@ query: query: range: http.clientip: - from: 232.0.0.0 - to: 253.0.0.0 + gte: 232.0.0.0 + lte: 253.0.0.0 - match: { hits.total.value: 4 } --- From bfe1318130e9aa415dc6ce61ad13b6e0717042f9 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Mon, 23 Sep 2024 10:02:23 -0400 Subject: [PATCH 08/10] Skip compatibility tests as warning don't exist for compatibility --- modules/rank-eval/build.gradle | 4 ++++ modules/runtime-fields-common/build.gradle | 4 ++++ rest-api-spec/build.gradle | 2 ++ x-pack/plugin/build.gradle | 4 ++++ 4 files changed, 14 insertions(+) diff --git a/modules/rank-eval/build.gradle b/modules/rank-eval/build.gradle index c9016798c18b9..511dd7be9ae68 100644 --- a/modules/rank-eval/build.gradle +++ b/modules/rank-eval/build.gradle @@ -25,3 +25,7 @@ testClusters.configureEach { // Modules who's integration is explicitly tested in integration tests module ':modules:lang-mustache' } + +tasks.named("yamlRestCompatTestTransform").configure({ task -> + task.skipTest("rank_eval/30_failures/Response format", "warning does not exist for compatibility") +}) diff --git a/modules/runtime-fields-common/build.gradle b/modules/runtime-fields-common/build.gradle index 411790cb04f51..37e250adb3bc5 100644 --- a/modules/runtime-fields-common/build.gradle +++ b/modules/runtime-fields-common/build.gradle @@ -22,3 +22,7 @@ dependencies { api project(':libs:elasticsearch-grok') api project(':libs:elasticsearch-dissect') } + +tasks.named("yamlRestCompatTestTransform").configure({ task -> + task.skipTest("runtime_fields/110_composite/query", "warning does not exist for compatibility") +}) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index eb3d66ece50d6..5ea0b202d76de 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -56,4 +56,6 @@ tasks.named("precommit").configure { tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("tsdb/140_routing_path/multi-value routing path field", "Multi-value routing paths are allowed now. See #112645") + task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility") + task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility") }) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 4fdc4c3af4190..ebc79ca6ce44a 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -80,3 +80,7 @@ tasks.named("precommit").configure { dependsOn 'enforceYamlTestConvention', 'enforceApiSpecsConvention' } +tasks.named("yamlRestCompatTestTransform").configure({ task -> + task.skipTest("security/10_forbidden/Test bulk response with invalid credentials", "warning does not exist for compatibility") +}) + From 8971beba67ce2bc9ed8f7b4966e172e8b9b9a8c7 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Mon, 23 Sep 2024 14:15:30 -0400 Subject: [PATCH 09/10] Skipping test for Rest compatibility --- modules/runtime-fields-common/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/runtime-fields-common/build.gradle b/modules/runtime-fields-common/build.gradle index 37e250adb3bc5..00bb17df8665e 100644 --- a/modules/runtime-fields-common/build.gradle +++ b/modules/runtime-fields-common/build.gradle @@ -24,5 +24,5 @@ dependencies { } tasks.named("yamlRestCompatTestTransform").configure({ task -> - task.skipTest("runtime_fields/110_composite/query", "warning does not exist for compatibility") + task.skipTestsByFilePattern("**/runtime_fields/110_composite.yml", "warning does not exist for compatibility") }) From eafb33c24faa7dc9af27ab99b014baca4e31fadf Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Mon, 23 Sep 2024 16:21:20 -0400 Subject: [PATCH 10/10] Temporarily mute test for BWC --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 0ceaeff67e2c5..123486cd5373d 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -301,6 +301,9 @@ tests: - class: org.elasticsearch.xpack.ml.integration.MlJobIT method: testDeleteJob_TimingStatsDocumentIsDeleted issue: https://github.com/elastic/elasticsearch/issues/113370 +- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT + method: test {p0=search/500_date_range/from, to, include_lower, include_upper deprecated} + issue: https://github.com/elastic/elasticsearch/pull/113286 # Examples: #