diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index 2279bb4acc1a8..ebd788383bff5 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -54,6 +54,7 @@ tasks.named("yamlRestTest").configure { systemProperty 'tests.rest.blacklist', [ // for some reason, allowed_warnings on the test isn't working here 'ingest/30_date_processor/Test date processor with no timezone configured', + 'ingest/30_date_processor/Test week based date parsing', ].join(',') } @@ -61,6 +62,7 @@ tasks.named("yamlRestTestV7CompatTest").configure { systemProperty 'tests.rest.blacklist', [ // for some reason, allowed_warnings on the test isn't working here 'ingest/30_date_processor/Test date processor with no timezone configured', + 'ingest/30_date_processor/Test week based date parsing', ].join(',') } diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml index 8508534614c4b..b2710ff05aa00 100644 --- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml @@ -211,6 +211,8 @@ teardown: --- "Test week based date parsing": - do: + allowed_warnings: + - 'Date format [YYYY-ww] contains week-date field specifiers that are changing in JDK 23' indices.create: index: test body: @@ -221,6 +223,8 @@ teardown: format: YYYY-ww - do: + allowed_warnings: + - 'Date format [YYYY-ww] contains week-date field specifiers that are changing in JDK 23' ingest.put_pipeline: id: "my_pipeline" body: > @@ -270,69 +274,3 @@ teardown: id: "1" - match: { _source.date_source_field: "2020-33" } - match: { _source.date_target_field: "2020-08-10T00:00:00.000Z" } - ---- -"Test week based date parsing with locale": - #locale is used when parsing as well on a pipeline. As per US locale, start of the 33rd week 2020 is on 09August2020 (sunday) - - do: - indices.create: - index: test - body: - mappings: - properties: - date_source_field: - type: date - format: YYYY-ww - locale: en-US - - - do: - ingest.put_pipeline: - id: "my_pipeline" - body: > - { - "description": "_description", - "processors": [ - { - "date" : { - "field" : "date_source_field", - "target_field" : "date_target_field", - "formats" : ["YYYY-ww"], - "locale" : "en-US" - } - } - ] - } - - match: { acknowledged: true } - - - do: - ingest.simulate: - id: "my_pipeline" - body: > - { - "docs": [ - { - "_source": { - "date_source_field": "2020-33" - } - } - ] - } - - length: { docs: 1 } - - match: { docs.0.doc._source.date_source_field: "2020-33" } - - match: { docs.0.doc._source.date_target_field: "2020-08-09T00:00:00.000Z" } - - length: { docs.0.doc._ingest: 1 } - - is_true: docs.0.doc._ingest.timestamp - - - do: - index: - index: test - id: "1" - pipeline: "my_pipeline" - body: {date_source_field: "2020-33"} - - - do: - get: - index: test - id: "1" - - match: { _source.date_source_field: "2020-33" } - - match: { _source.date_target_field: "2020-08-09T00:00:00.000Z" } diff --git a/server/src/main/java/org/elasticsearch/common/time/DateUtils.java b/server/src/main/java/org/elasticsearch/common/time/DateUtils.java index 91baaedf4a669..3e383db8503cf 100644 --- a/server/src/main/java/org/elasticsearch/common/time/DateUtils.java +++ b/server/src/main/java/org/elasticsearch/common/time/DateUtils.java @@ -387,10 +387,16 @@ public static ZonedDateTime nowWithMillisResolution(Clock clock) { return ZonedDateTime.now(millisResolutionClock); } + private static final boolean USES_COMPAT = System.getProperty("java.locale.providers", "").contains("COMPAT"); // check for all textual fields, and localized zone offset // the weird thing with Z is to ONLY match 4 in a row, with no Z before or after (but those groups can also be empty) - private static final Predicate CONTAINS_CHANGING_TEXT_SPECIFIERS = System.getProperty("java.locale.providers", "") - .contains("COMPAT") ? Pattern.compile("[BEGOavz]|LLL|MMM|QQQ|qqq|ccc|eee|(? CONTAINS_CHANGING_TEXT_SPECIFIERS = USES_COMPAT + ? Pattern.compile("[BEGOavz]|LLL|MMM|QQQ|qqq|ccc|eee|(? CONTAINS_WEEK_DATE_SPECIFIERS = USES_COMPAT + ? Pattern.compile("[YWw]").asPredicate() + : Predicates.never(); @UpdateForV9 // this can be removed, we will only use CLDR on v9 static void checkTextualDateFormats(String format) { @@ -402,5 +408,13 @@ static void checkTextualDateFormats(String format) { format ); } + if (CONTAINS_WEEK_DATE_SPECIFIERS.test(format)) { + deprecationLogger.warn( + DeprecationCategory.PARSING, + "cldr_week_dates_" + format, + "Date format [{}] contains week-date field specifiers that are changing in JDK 23", + format + ); + } } } diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index ff45cc9430633..1e1d25363f73d 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -77,6 +77,9 @@ tasks.named("yamlRestTestV7CompatTest").configure { 'ml/evaluate_data_frame/Test outlier_detection auc_roc include curve', 'ml/evaluate_data_frame/Test classification auc_roc', 'ml/evaluate_data_frame/Test classification auc_roc with default top_classes_field', + 'security/authz/13_index_datemath/Test indexing documents with datemath, when forbidden', + 'security/authz/13_index_datemath/Test indexing documents with datemath, when permitted', + 'security/authz/13_index_datemath/Test bulk indexing with datemath when only some are allowed', ].join(',') } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec index fd25f53a1108b..2ed6ffe0dbca1 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec @@ -45,7 +45,7 @@ emp_no:integer | x:date evalDateFormat -from employees | sort hire_date | eval x = date_format(hire_date), y = date_format("YYYY-MM-dd", hire_date) | keep emp_no, x, y | limit 5; +from employees | sort hire_date | eval x = date_format(hire_date), y = date_format("yyyy-MM-dd", hire_date) | keep emp_no, x, y | limit 5; emp_no:integer | x:keyword | y:keyword 10009 | 1985-02-18T00:00:00.000Z | 1985-02-18 @@ -882,7 +882,7 @@ docsDateFormat // tag::docsDateFormat[] FROM employees | KEEP first_name, last_name, hire_date -| EVAL hired = DATE_FORMAT("YYYY-MM-dd", hire_date) +| EVAL hired = DATE_FORMAT("yyyy-MM-dd", hire_date) // end::docsDateFormat[] | SORT first_name | LIMIT 3 @@ -900,7 +900,7 @@ evalDateFormatString required_capability: string_literal_auto_casting ROW a = 1 -| EVAL df = DATE_FORMAT("YYYY-MM-dd", "1989-06-02T00:00:00.000Z") +| EVAL df = DATE_FORMAT("yyyy-MM-dd", "1989-06-02T00:00:00.000Z") ; a:integer | df:keyword diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec index 923b3b8dacff1..a85517ccdad6a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec @@ -926,7 +926,7 @@ avg_lang:double | max_lang:integer docsStatsGroupByMultipleValues // tag::statsGroupByMultipleValues[] FROM employees -| EVAL hired = DATE_FORMAT("YYYY", hire_date) +| EVAL hired = DATE_FORMAT("yyyy", hire_date) | STATS avg_salary = AVG(salary) BY hired, languages.long | EVAL avg_salary = ROUND(avg_salary) | SORT hired, languages.long diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/13_index_datemath.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/13_index_datemath.yml index db1dd72553b3f..eee0e273974bd 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/13_index_datemath.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/13_index_datemath.yml @@ -1,5 +1,7 @@ --- setup: + - requires: + test_runner_features: allowed_warnings - skip: features: headers @@ -54,6 +56,8 @@ teardown: } - do: + allowed_warnings: + - 'Date format [YYYY.MM] contains week-date field specifiers that are changing in JDK 23' headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user bulk: body: @@ -89,6 +93,8 @@ teardown: } - do: + allowed_warnings: + - 'Date format [YYYY.MM] contains week-date field specifiers that are changing in JDK 23' headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user bulk: body: @@ -115,6 +121,8 @@ teardown: --- "Test bulk indexing with datemath when only some are allowed": - do: + allowed_warnings: + - 'Date format [YYYY] contains week-date field specifiers that are changing in JDK 23' headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user bulk: body: