diff --git a/ydb/library/yql/providers/solomon/actors/dq_solomon_metrics_queue.cpp b/ydb/library/yql/providers/solomon/actors/dq_solomon_metrics_queue.cpp index 369996b5297b..69f0013f24e9 100644 --- a/ydb/library/yql/providers/solomon/actors/dq_solomon_metrics_queue.cpp +++ b/ydb/library/yql/providers/solomon/actors/dq_solomon_metrics_queue.cpp @@ -235,6 +235,8 @@ class TDqSolomonMetricsQueueActor : public NActors::TActorBootstrapped= response.Result.PagesCount) { LOG_I("TDqSolomonMetricsQueueActor", "SaveRetrievedResults no more metrics to list"); HasMoreMetrics = false; @@ -372,7 +374,8 @@ class TDqSolomonMetricsQueueActor : public NActors::TActorBootstrapped> PendingRequests; std::vector Metrics; + ui64 DownloadedBytes = 0; TMaybe MaybeIssues; const TDqSolomonReadParams ReadParams; diff --git a/ydb/library/yql/providers/solomon/actors/dq_solomon_read_actor.cpp b/ydb/library/yql/providers/solomon/actors/dq_solomon_read_actor.cpp index db625b0ecb38..59d1c91ed76f 100644 --- a/ydb/library/yql/providers/solomon/actors/dq_solomon_read_actor.cpp +++ b/ydb/library/yql/providers/solomon/actors/dq_solomon_read_actor.cpp @@ -204,6 +204,9 @@ class TDqSolomonReadActor : public NActors::TActorBootstrapped to) { + if (timestamp < from || timestamp >= to) { continue; } @@ -388,6 +395,10 @@ class TDqSolomonReadActor : public NActors::TActorBootstrapped { TEvMetricsBatch() = default; - TEvMetricsBatch(std::vector metrics, bool noMoreMetrics, const NDqProto::TMessageTransportMeta& transportMeta) { + TEvMetricsBatch(std::vector metrics, bool noMoreMetrics, ui64 downloadedBytes, const NDqProto::TMessageTransportMeta& transportMeta) { Record.MutableMetrics()->Assign( metrics.begin(), metrics.end()); Record.SetNoMoreMetrics(noMoreMetrics); + Record.SetDownloadedBytes(downloadedBytes); *Record.MutableTransportMeta() = transportMeta; } }; diff --git a/ydb/library/yql/providers/solomon/proto/metrics_queue.proto b/ydb/library/yql/providers/solomon/proto/metrics_queue.proto index a658b348d37a..0d3aea371d9b 100644 --- a/ydb/library/yql/providers/solomon/proto/metrics_queue.proto +++ b/ydb/library/yql/providers/solomon/proto/metrics_queue.proto @@ -35,6 +35,7 @@ message TEvGetNextBatch { message TEvMetricsBatch { bool NoMoreMetrics = 1; repeated TMetric Metrics = 2; + uint64 DownloadedBytes = 3; optional NYql.NDqProto.TMessageTransportMeta TransportMeta = 100; } diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp index 7a165b3c073a..d0c862edbb1e 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp @@ -136,10 +136,12 @@ class TSolomonDqIntegration: public TDqIntegrationBase { if (!ExtractSettingValue(settingsRef.Child(i)->Tail(), settingsRef.Child(i)->Head().Content(), ctx, value)) { return {}; } - if (!TInstant::TryParseIso8601(value, from)) { - ctx.AddError(TIssue(ctx.GetPosition(settingsRef.Child(i)->Head().Pos()), "couldn't parse `from`, use Iso8601 format, e.g. 2025-03-12T14:40:39Z")); + TInstant userFrom; + if (!TInstant::TryParseIso8601(value, userFrom)) { + ctx.AddError(TIssue(ctx.GetPosition(settingsRef.Child(i)->Head().Pos()), "couldn't parse `from`, use ISO8601 format, e.g. 2025-03-12T14:40:39Z")); return {}; } + from = std::min(TInstant::Now(), userFrom); continue; } if (settingsRef.Child(i)->Head().IsAtom("to"sv)) { @@ -147,10 +149,12 @@ class TSolomonDqIntegration: public TDqIntegrationBase { if (!ExtractSettingValue(settingsRef.Child(i)->Tail(), settingsRef.Child(i)->Head().Content(), ctx, value)) { return {}; } - if (!TInstant::TryParseIso8601(value, to)) { - ctx.AddError(TIssue(ctx.GetPosition(settingsRef.Child(i)->Head().Pos()), "couldn't parse `to`, use Iso8601 format, e.g. 2025-03-12T14:40:39Z")); + TInstant userTo; + if (!TInstant::TryParseIso8601(value, userTo)) { + ctx.AddError(TIssue(ctx.GetPosition(settingsRef.Child(i)->Head().Pos()), "couldn't parse `to`, use ISO8601 format, e.g. 2025-03-12T14:40:39Z")); return {}; } + to = std::min(TInstant::Now(), userTo); continue; } if (settingsRef.Child(i)->Head().IsAtom("program"sv)) { diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp index bce982292cea..833e0a4b521a 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp @@ -69,14 +69,20 @@ class TSolomonLoadTableMetadataTransformer : public TGraphTransformerBase { TInstant from; if (auto time = ExtractSetting(settings, "from")) { - from = TInstant::ParseIso8601(*time); + if (!TInstant::TryParseIso8601(*time, from)) { + ctx.AddError(TIssue(ctx.GetPosition(n->Pos()), "couldn't parse `from`, use ISO8601 format, e.g. 2025-03-12T14:40:39Z")); + return TStatus::Error; + } } else { - from = TInstant::Now() - TDuration::Days(7); + from = TInstant::Zero(); } TInstant to; if (auto time = ExtractSetting(settings, "to")) { - to = TInstant::ParseIso8601(*time); + if (!TInstant::TryParseIso8601(*time, to)) { + ctx.AddError(TIssue(ctx.GetPosition(n->Pos()), "couldn't parse `to`, use ISO8601 format, e.g. 2025-03-12T14:40:39Z")); + return TStatus::Error; + } } else { to = TInstant::Now(); } diff --git a/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.cpp b/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.cpp index 89f53116d0d0..e5a9f6b1a391 100644 --- a/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.cpp +++ b/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.cpp @@ -74,67 +74,70 @@ TGetLabelsResponse ProcessGetLabelsResponse(NYql::IHTTPGateway::TResult&& respon TGetLabelsResult result; if (response.CurlResponseCode != CURLE_OK) { - return TGetLabelsResponse(TStringBuilder{} << "Error while sending list metric names request to monitoring api: " << response.Issues.ToOneLineString()); + return TGetLabelsResponse(TStringBuilder{} << "Monitoring api get labels response: " << response.Issues.ToOneLineString() << + ", internal code: " << static_cast(response.CurlResponseCode)); } if (response.Content.HttpResponseCode < 200 || response.Content.HttpResponseCode >= 300) { - return TGetLabelsResponse(TStringBuilder{} << "Error while sending list metric names request to monitoring api: " << response.Content.data()); + return TGetLabelsResponse(TStringBuilder{} << "Monitoring api get labels response: " << response.Content.data() << + ", internal code: " << response.Content.HttpResponseCode); } NJson::TJsonValue json; try { NJson::ReadJsonTree(response.Content.data(), &json, /*throwOnError*/ true); } catch (const std::exception& e) { - return TGetLabelsResponse(TStringBuilder{} << "Failed to parse response from monitoring api: " << e.what()); + return TGetLabelsResponse("Monitoring api get labels response is not a valid json"); } if (!json.IsMap() || !json.Has("names") || !json["names"].IsArray()) { - return TGetLabelsResponse("Invalid result from monitoring api"); + return TGetLabelsResponse("Monitoring api get labels response doesn't contain requested info"); } const auto names = json["names"].GetArray(); for (const auto& name : names) { if (!name.IsString()) { - return TGetLabelsResponse("Invalid label names from monitoring api"); - } else { - result.Labels.push_back(name.GetString()); + return TGetLabelsResponse("Monitoring api get labels response contains invalid label names"); } + result.Labels.push_back(name.GetString()); } for (const auto& [key, selector] : knownSelectors) { result.Labels.push_back(key); } - return TGetLabelsResponse(std::move(result)); + return TGetLabelsResponse(std::move(result), response.Content.size() + response.Content.Headers.size()); } TListMetricsResponse ProcessListMetricsResponse(NYql::IHTTPGateway::TResult&& response) { TListMetricsResult result; if (response.CurlResponseCode != CURLE_OK) { - return TListMetricsResponse(TStringBuilder{} << "Error while sending list metrics request to monitoring api: " << response.Issues.ToOneLineString()); + return TListMetricsResponse(TStringBuilder{} << "Monitoring api list metrics response: " << response.Issues.ToOneLineString() << + ", internal code: " << static_cast(response.CurlResponseCode)); } if (response.Content.HttpResponseCode < 200 || response.Content.HttpResponseCode >= 300) { - return TListMetricsResponse(TStringBuilder{} << "Error while sending list metrics request to monitoring api: " << response.Content.data()); + return TListMetricsResponse(TStringBuilder{} << "Monitoring api list metrics response: " << response.Content.data() << + ", internal code: " << response.Content.HttpResponseCode); } NJson::TJsonValue json; try { NJson::ReadJsonTree(response.Content.data(), &json, /*throwOnError*/ true); } catch (const std::exception& e) { - return TListMetricsResponse(TStringBuilder{} << "Failed to parse response from monitoring api: " << e.what()); + return TListMetricsResponse("Monitoring api list metrics response is not a valid json" ); } if (!json.IsMap() || !json.Has("result") || !json.Has("page")) { - return TListMetricsResponse("Invalid list metrics result from monitoring api"); + return TListMetricsResponse("Monitoring api list metrics response doesn't contain requested info"); } const auto pagesInfo = json["page"]; if (!pagesInfo.IsMap() || !pagesInfo.Has("pagesCount") || !pagesInfo["pagesCount"].IsInteger() || !pagesInfo.Has("totalCount") || !pagesInfo["totalCount"].IsInteger()) { - return TListMetricsResponse("Invalid paging info from monitoring api"); + return TListMetricsResponse("Monitoring api list metrics response doesn't contain paging info"); } result.PagesCount = pagesInfo["pagesCount"].GetInteger(); @@ -142,7 +145,7 @@ TListMetricsResponse ProcessListMetricsResponse(NYql::IHTTPGateway::TResult&& re for (const auto& metricObj : json["result"].GetArray()) { if (!metricObj.IsMap() || !metricObj.Has("labels") || !metricObj["labels"].IsMap() || !metricObj.Has("type") || !metricObj["type"].IsString()) { - return TListMetricsResponse("Invalid list metrics result from monitoring api"); + return TListMetricsResponse("Monitoring api list metrics response contains invalid metrics"); } TSelectors selectors; @@ -153,7 +156,7 @@ TListMetricsResponse ProcessListMetricsResponse(NYql::IHTTPGateway::TResult&& re result.Metrics.emplace_back(std::move(selectors), metricObj["type"].GetString()); } - return TListMetricsResponse(std::move(result)); + return TListMetricsResponse(std::move(result), response.Content.size() + response.Content.Headers.size()); } TGetPointsCountResponse ProcessGetPointsCountResponse(NYql::IHTTPGateway::TResult&& response, ui64 downsampledPointsCount) { @@ -169,38 +172,40 @@ TGetPointsCountResponse ProcessGetPointsCountResponse(NYql::IHTTPGateway::TResul for (const auto& whitelistIssue : whitelistIssues) { if (issues.find(whitelistIssue) != issues.npos) { result.PointsCount = 0; - return TGetPointsCountResponse(std::move(result)); + return TGetPointsCountResponse(std::move(result), 0); } } - return TGetPointsCountResponse(TStringBuilder() << "Error while sending points count request to monitoring api: " << issues); + return TGetPointsCountResponse(TStringBuilder() << "Monitoring api points count response: " << issues << + ", internal code: " << static_cast(response.CurlResponseCode)); } if (response.Content.HttpResponseCode < 200 || response.Content.HttpResponseCode >= 300) { - return TGetPointsCountResponse(TStringBuilder{} << "Error while sending points count request to monitoring api: " << response.Content.data()); + return TGetPointsCountResponse(TStringBuilder{} << "Monitoring api points count response: " << response.Content.data() << + ", internal code: " << response.Content.HttpResponseCode); } NJson::TJsonValue json; try { NJson::ReadJsonTree(response.Content.data(), &json, /*throwOnError*/ true); } catch (const std::exception& e) { - return TGetPointsCountResponse(TStringBuilder{} << "Failed to parse points count response from monitoring api: " << e.what()); + return TGetPointsCountResponse("Monitoring api points count response is not a valid json"); } if (!json.IsMap() || !json.Has("scalar") || !json["scalar"].IsInteger()) { - return TGetPointsCountResponse("Invalid points count result from monitoring api"); + return TGetPointsCountResponse("Monitoring api points count response doesn't contain requested info"); } result.PointsCount = json["scalar"].GetInteger() + downsampledPointsCount; - return TGetPointsCountResponse(std::move(result)); + return TGetPointsCountResponse(std::move(result), response.Content.size() + response.Content.Headers.size()); } TGetDataResponse ProcessGetDataResponse(NYdbGrpc::TGrpcStatus&& status, ReadResponse&& response) { TGetDataResult result; if (!status.Ok()) { - TString error = TStringBuilder{} << "Error while sending data request to monitoring api: " << status.Msg; + TString error = TStringBuilder{} << "Monitoring api get data response: " << status.Msg; if (status.GRpcStatusCode == grpc::StatusCode::RESOURCE_EXHAUSTED || status.GRpcStatusCode == grpc::StatusCode::UNAVAILABLE) { return TGetDataResponse(error, EStatus::STATUS_RETRIABLE_ERROR); } @@ -208,7 +213,7 @@ TGetDataResponse ProcessGetDataResponse(NYdbGrpc::TGrpcStatus&& status, ReadResp } if (response.response_per_query_size() != 1) { - return TGetDataResponse("Invalid get data repsonse size from monitoring api"); + return TGetDataResponse("Monitoring api get data response is invalid"); } const auto& responseValue = response.response_per_query()[0]; @@ -235,7 +240,7 @@ TGetDataResponse ProcessGetDataResponse(NYdbGrpc::TGrpcStatus&& status, ReadResp result.Timeseries.emplace_back(std::move(metric), std::move(timestamps), std::move(values)); } - return TGetDataResponse(std::move(result)); + return TGetDataResponse(std::move(result), response.ByteSize()); } class TSolomonAccessorClient : public ISolomonAccessorClient, public std::enable_shared_from_this { @@ -329,7 +334,7 @@ class TSolomonAccessorClient : public ISolomonAccessorClient, public std::enable TGetPointsCountResult result; result.PointsCount = downsampledPointsCount; - resultPromise.SetValue(TGetPointsCountResponse(std::move(result))); + resultPromise.SetValue(TGetPointsCountResponse(std::move(result), 0)); } return resultPromise.GetFuture(); @@ -579,7 +584,7 @@ class TSolomonAccessorClient : public ISolomonAccessorClient, public std::enable fullSelectors[labelName] = {"=", "-"}; } } - return selectors; + return fullSelectors; } TString BuildSelectorsProgram(const TSelectors& selectors, bool useNewFormat = false) const { diff --git a/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_client_utils.cpp b/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_client_utils.cpp index f67c7ef0fc60..32b3437e88a6 100644 --- a/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_client_utils.cpp +++ b/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_client_utils.cpp @@ -18,9 +18,10 @@ TSolomonClientResponse::TSolomonClientResponse(const TString& error, EStatus , Error(error) {} template -TSolomonClientResponse::TSolomonClientResponse(T&& result) +TSolomonClientResponse::TSolomonClientResponse(T&& result, ui64 downloadedBytes) : Status(STATUS_OK) - , Result(std::move(result)) {} + , Result(std::move(result)) + , DownloadedBytes(downloadedBytes) {} template class TSolomonClientResponse; template class TSolomonClientResponse; diff --git a/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_client_utils.h b/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_client_utils.h index 08269cb85544..3eb5fa067ad7 100644 --- a/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_client_utils.h +++ b/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_client_utils.h @@ -20,12 +20,13 @@ class TSolomonClientResponse { public: TSolomonClientResponse(); explicit TSolomonClientResponse(const TString& error, EStatus status = STATUS_FATAL_ERROR); - explicit TSolomonClientResponse(T&& result); + explicit TSolomonClientResponse(T&& result, ui64 downloadedBytes); public: EStatus Status; TString Error; T Result; + ui64 DownloadedBytes = 0; }; struct TGetLabelsResult { diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-InvalidProject-_/extracted b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-InvalidProject-_/extracted index 84523bbd1f46..02e0d15e70f0 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-InvalidProject-_/extracted +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-InvalidProject-_/extracted @@ -6,4 +6,4 @@ /program.sql:
: Error: Fatal Error - /program.sql:
: Error: Error while sending data request to monitoring api: Project invalid does not exist (appeared 1 time at ISOTIME) \ No newline at end of file + /program.sql:
: Error: Monitoring api get data response: Project invalid does not exist (appeared 1 time at ISOTIME) \ No newline at end of file diff --git a/ydb/tests/fq/solomon/canondata/test.test_solomon-InvalidProject-_/extracted b/ydb/tests/fq/solomon/canondata/test.test_solomon-InvalidProject-_/extracted index 85ad101b4850..c51f09a2d0c9 100644 --- a/ydb/tests/fq/solomon/canondata/test.test_solomon-InvalidProject-_/extracted +++ b/ydb/tests/fq/solomon/canondata/test.test_solomon-InvalidProject-_/extracted @@ -1,7 +1,7 @@ Failed to execute query, reason: Request finished with status: EXTERNAL_ERROR Issues: -
: Error: Error while sending data request to monitoring api: Project invalid does not exist +
: Error: Monitoring api get data response: Project invalid does not exist diff --git a/ydb/tests/solomon/reading/base.py b/ydb/tests/solomon/reading/base.py index 56aac355fa8c..b2e10dc2e945 100644 --- a/ydb/tests/solomon/reading/base.py +++ b/ydb/tests/solomon/reading/base.py @@ -13,8 +13,8 @@ class SolomonReadingTestBase(object): @classmethod def setup_class(cls): - cls.basic_reading_timestamps = [0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60] - cls.basic_reading_values = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + cls.basic_reading_timestamps = [0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55] + cls.basic_reading_values = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] cls.listing_paging_metrics_size = 1000 @@ -23,6 +23,15 @@ def setup_class(cls): cleanup_emulator() + add_solomon_metrics("settings_validation", "settings_validation", "my_service", {"metrics": [ + { + "labels" : {"test_type": "setting_validation"}, + "type" : "DGAUGE", + "timestamps" : [1000000], + "values" : [0] + } + ]}) + add_solomon_metrics("basic_reading", "basic_reading", "my_service", {"metrics": [ { "labels" : {"test_type": "basic_reading_test"}, @@ -96,7 +105,7 @@ def execute_query(cls, query): @staticmethod def _generate_listing_paging_test_metrics(size): - return [ + listing_paging_metrics = [ { "labels" : {"test_type": "listing_paging_test", "test_label": str(i)}, "type" : "DGAUGE", @@ -106,6 +115,15 @@ def _generate_listing_paging_test_metrics(size): for i in range(size) ] + listing_paging_metrics.append({ + "labels" : {"test_type": "listing_paging_test"}, + "type" : "DGAUGE", + "timestamps" : [0], + "values" : [0] + }) + + return listing_paging_metrics + @staticmethod def _generate_data_paging_timeseries(size): timestamps = [i * 5 for i in range(size)] diff --git a/ydb/tests/solomon/reading/basic_reading.py b/ydb/tests/solomon/reading/basic_reading.py index 60b30fcf5ac2..9fc401c942ea 100644 --- a/ydb/tests/solomon/reading/basic_reading.py +++ b/ydb/tests/solomon/reading/basic_reading.py @@ -27,6 +27,17 @@ def check_query_result(self, result, error, downsampling_disabled): return False, "values differ from canonical, have {}, should be {}".format(values, canon_values) return True, None + def check_query_result_size(self, result, error): + if error is not None: + return False, error + + result_size = len(result[0].rows) + + if (result_size != 1): + return False, "should only have a single return row, have: {}".format(result_size) + + return True, None + @link_test_case("#16398") def test_basic_reading_solomon(self): data_source_query = f""" @@ -144,6 +155,18 @@ def test_basic_reading_solomon(self): assert error is None, error assert any(column.name == "tt" for column in result[0].columns) + # query with a single second interval + query = """ + SELECT * FROM local_solomon.basic_reading WITH ( + selectors = @@{cluster="basic_reading", service="my_service", test_type="basic_reading_test"}@@, + + from = "1970-01-01T00:00:00Z", + to = "1970-01-01T00:00:01Z" + ) + """ + succes, error = self.check_query_result_size(*self.execute_query(query)) + assert succes, error + @link_test_case("#23192") def test_basic_reading_monitoring(self): data_source_query = f""" @@ -262,3 +285,15 @@ def test_basic_reading_monitoring(self): result, error = self.execute_query(query) assert error is None, error assert any(column.name == "tt" for column in result[0].columns) + + # query with a single second interval + query = """ + SELECT * FROM local_monitoring.my_service WITH ( + selectors = @@{test_type="basic_reading_test"}@@, + + from = "1970-01-01T00:00:00Z", + to = "1970-01-01T00:00:01Z" + ) + """ + succes, error = self.check_query_result_size(*self.execute_query(query)) + assert succes, error diff --git a/ydb/tests/solomon/reading/listing_paging.py b/ydb/tests/solomon/reading/listing_paging.py index 2dac0f0c51db..20c39c93fb7c 100644 --- a/ydb/tests/solomon/reading/listing_paging.py +++ b/ydb/tests/solomon/reading/listing_paging.py @@ -7,7 +7,7 @@ class TestListingPaging(SolomonReadingTestBase): - def check_listing_paging_result(self, result_set, error): + def check_full_listing_result(self, result_set, error): if error is not None: return False, error @@ -29,6 +29,19 @@ def check_listing_paging_result(self, result_set, error): return True, None + def check_listing_size(self, result_set, error): + if error is not None: + return False, error + + rows = [] + for result in result_set: + rows.extend(result.rows) + + if (len(rows) != self.listing_paging_metrics_size + 1): + return False, "Result size differs from expected: have {}, should be {}".format(len(rows), self.listing_paging_metrics_size) + + return True, None + @link_test_case("#16395") def test_listing_paging_solomon(self): data_source_query = f""" @@ -42,7 +55,6 @@ def test_listing_paging_solomon(self): result, error = self.execute_query(data_source_query) assert error is None - # simplest query with default downsampling settings query = """ SELECT test_label FROM local_solomon.listing_paging WITH ( selectors = @@{cluster="listing_paging", service="my_service", test_type="listing_paging_test", test_label="*"}@@, @@ -53,7 +65,18 @@ def test_listing_paging_solomon(self): to = "1970-01-01T00:01:00Z" ) """ - success, error = self.check_listing_paging_result(*self.execute_query(query)) + success, error = self.check_full_listing_result(*self.execute_query(query)) + assert success, error + + query = """ + SELECT * FROM local_solomon.listing_paging WITH ( + selectors = @@{cluster="listing_paging", service="my_service", test_type="listing_paging_test"}@@, + + from = "1970-01-01T00:00:00Z", + to = "1970-01-01T00:01:00Z" + ) + """ + success, error = self.check_listing_size(*self.execute_query(query)) assert success, error @link_test_case("#23190") @@ -71,7 +94,6 @@ def test_listing_paging_monitoring(self): result, error = self.execute_query(data_source_query) assert error is None - # simplest query with default downsampling settings query = """ SELECT test_label FROM local_monitoring.my_service WITH ( selectors = @@{test_type="listing_paging_test", test_label="*"}@@, @@ -82,5 +104,16 @@ def test_listing_paging_monitoring(self): to = "1970-01-01T00:01:00Z" ) """ - success, error = self.check_listing_paging_result(*self.execute_query(query)) + success, error = self.check_full_listing_result(*self.execute_query(query)) + assert success, error + + query = """ + SELECT * FROM local_monitoring.my_service WITH ( + selectors = @@{test_type="listing_paging_test"}@@, + + from = "1970-01-01T00:00:00Z", + to = "1970-01-01T00:01:00Z" + ) + """ + success, error = self.check_listing_size(*self.execute_query(query)) assert success, error diff --git a/ydb/tests/solomon/reading/settings_validation.py b/ydb/tests/solomon/reading/settings_validation.py index da592f6a0515..9cd81ffcdba3 100644 --- a/ydb/tests/solomon/reading/settings_validation.py +++ b/ydb/tests/solomon/reading/settings_validation.py @@ -21,7 +21,7 @@ def check_query_error(self, query, error_msg): assert error_msg in extract_issue_messages(error), "Expected to find specific error: {}, have errors: {}".format(error_msg, error) @link_test_case("#16385") - def test_settings_validation_solomon(self): + def test_settings_validation_solomon_selectors(self): data_source_query = f""" CREATE EXTERNAL DATA SOURCE local_solomon WITH ( SOURCE_TYPE = "Solomon", @@ -84,6 +84,139 @@ def test_settings_validation_solomon(self): """ self.check_query_error(query, "Label names should be specified in \"label1 [as alias1], label2 [as alias2], ...\" format") + # check `from` setting validation + query = """ + SELECT * FROM local_solomon.settings_validation WITH ( + selectors = @@{cluster="settings_validation", service="my_service", test_type="setting_validation"}@@, + from = "invalid time" + ) + """ + self.check_query_error(query, "couldn\'t parse `from`, use ISO8601 format") + + # check `to` setting validation + query = """ + SELECT * FROM local_solomon.settings_validation WITH ( + selectors = @@{cluster="settings_validation", service="my_service", test_type="setting_validation"}@@, + to = "invalid time" + ) + """ + self.check_query_error(query, "couldn\'t parse `to`, use ISO8601 format") + + # check `downsampling.disabled` setting validation + query = """ + SELECT * FROM local_solomon.settings_validation WITH ( + selectors = @@{cluster="settings_validation", service="my_service", test_type="setting_validation"}@@, + `downsampling.disabled` = "ABC" + ) + """ + self.check_query_error(query, "downsampling.disabled must be true or false, but has ABC") + + # check `downsampling.aggregation` setting validation + query = """ + SELECT * FROM local_solomon.settings_validation WITH ( + selectors = @@{cluster="settings_validation", service="my_service", test_type="setting_validation"}@@, + `downsampling.aggregation` = "ABC" + ) + """ + self.check_query_error(query, "downsampling.aggregation must be one of AVG, COUNT, DEFAULT_AGGREGATION, LAST, MAX, MIN, SUM, but has ABC") + + # check `downsampling.fill` setting validation + query = """ + SELECT * FROM local_solomon.settings_validation WITH ( + selectors = @@{cluster="settings_validation", service="my_service", test_type="setting_validation"}@@, + `downsampling.fill` = "ABC" + ) + """ + self.check_query_error(query, "downsampling.fill must be one of NONE, NULL, PREVIOUS, but has ABC") + + # check `downsampling.grid_interval` setting validation + query = """ + SELECT * FROM local_solomon.settings_validation WITH ( + selectors = @@{cluster="settings_validation", service="my_service", test_type="setting_validation"}@@, + `downsampling.grid_interval` = "ABC" + ) + """ + self.check_query_error(query, "downsampling.grid_interval must be positive number, but has ABC") + + # check unknown setting validation + query = """ + SELECT * FROM local_solomon.settings_validation WITH ( + selectors = @@{cluster="settings_validation", service="my_service", test_type="setting_validation"}@@, + unk = "ABC" + ) + """ + self.check_query_error(query, "Unknown setting unk") + + # check additional downsampling settings validation + query = """ + SELECT * FROM local_solomon.settings_validation WITH ( + selectors = @@{cluster="settings_validation", service="my_service", test_type="setting_validation"}@@, + `downsampling.disabled` = "true", + `downsampling.aggregation` = "AVG", + `downsampling.fill` = "PREVIOUS", + `downsampling.grid_interval` = "15" + ) + """ + self.check_query_error(query, "downsampling.disabled must be false if downsampling.aggregation, downsampling.fill or downsampling.grid_interval is specified") + + query = """ + SELECT * FROM local_solomon.settings_validation WITH ( + selectors = @@{cluster="settings_validation", service="my_service", test_type="setting_validation"}@@, + + from = "1970-01-01T00:00:00Z", + to = "9999-01-01T00:00:01Z" + ) + """ + result, error = self.execute_query(query) + assert error is None + assert len(result[0].rows) == 0 + + drop_source_query = """ + DROP EXTERNAL DATA SOURCE local_solomon; + """ + result, error = self.execute_query(drop_source_query) + assert error is None + + @link_test_case("#16385") + def test_settings_validation_solomon_program(self): + data_source_query = f""" + CREATE EXTERNAL DATA SOURCE local_solomon WITH ( + SOURCE_TYPE = "Solomon", + LOCATION = "{self.solomon_http_endpoint}", + GRPC_LOCATION = "{self.solomon_grpc_endpoint}", + AUTH_METHOD = "NONE", + USE_TLS = "false" + ) + """ + result, error = self.execute_query(data_source_query) + assert error is None + + query = """ + SELECT * FROM local_solomon.settings_validation WITH ( + program = @@{cluster="settings_validation", service="my_service", test_type="setting_validation"}@@, + selectors = @@{cluster="settings_validation", service="my_service", test_type="setting_validation"}@@ + ) + """ + self.check_query_error(query, "either program or selectors must be specified") + + # check `labels` validation + query = """ + SELECT * FROM local_solomon.settings_validation WITH ( + program = @@{cluster="settings_validation", service="my_service", test_type="setting_validation"}@@, + labels = "[test_type]" + ) + """ + self.check_query_error(query, "Label names should be specified in \"label1 [as alias1], label2 [as alias2], ...\" format") + + # check `labels` validation + query = """ + SELECT * FROM local_solomon.settings_validation WITH ( + program = @@{cluster="settings_validation", service="my_service", test_type="setting_validation"}@@, + labels = @@"test_type"@@ + ) + """ + self.check_query_error(query, "Label names should be specified in \"label1 [as alias1], label2 [as alias2], ...\" format") + # check `from` setting validation query = """ SELECT * FROM local_solomon.settings_validation WITH ( @@ -91,7 +224,7 @@ def test_settings_validation_solomon(self): from = "invalid time" ) """ - self.check_query_error(query, "couldn\'t parse `from`, use Iso8601 format") + self.check_query_error(query, "couldn\'t parse `from`, use ISO8601 format") # check `to` setting validation query = """ @@ -100,7 +233,7 @@ def test_settings_validation_solomon(self): to = "invalid time" ) """ - self.check_query_error(query, "couldn\'t parse `to`, use Iso8601 format") + self.check_query_error(query, "couldn\'t parse `to`, use ISO8601 format") # check `downsampling.disabled` setting validation query = """ @@ -159,8 +292,14 @@ def test_settings_validation_solomon(self): """ self.check_query_error(query, "downsampling.disabled must be false if downsampling.aggregation, downsampling.fill or downsampling.grid_interval is specified") + drop_source_query = """ + DROP EXTERNAL DATA SOURCE local_solomon; + """ + result, error = self.execute_query(drop_source_query) + assert error is None + @link_test_case("#23189") - def test_settings_validation_monitoring(self): + def test_settings_validation_monitoring_selectors(self): data_source_query = f""" CREATE EXTERNAL DATA SOURCE local_monitoring WITH ( SOURCE_TYPE = "Solomon", @@ -225,6 +364,129 @@ def test_settings_validation_monitoring(self): """ self.check_query_error(query, "Label names should be specified in \"label1 [as alias1], label2 [as alias2], ...\" format") + # check `from` setting validation + query = """ + SELECT * FROM local_monitoring.my_service WITH ( + selectors = @@{test_type="setting_validation"}@@, + from = "invalid time" + ) + """ + self.check_query_error(query, "couldn\'t parse `from`, use ISO8601 format") + + # check `to` setting validation + query = """ + SELECT * FROM local_monitoring.my_service WITH ( + selectors = @@{test_type="setting_validation"}@@, + to = "invalid time" + ) + """ + self.check_query_error(query, "couldn\'t parse `to`, use ISO8601 format") + + # check `downsampling.disabled` setting validation + query = """ + SELECT * FROM local_monitoring.my_service WITH ( + selectors = @@{test_type="setting_validation"}@@, + `downsampling.disabled` = "ABC" + ) + """ + self.check_query_error(query, "downsampling.disabled must be true or false, but has ABC") + + # check `downsampling.aggregation` setting validation + query = """ + SELECT * FROM local_monitoring.my_service WITH ( + selectors = @@{test_type="setting_validation"}@@, + `downsampling.aggregation` = "ABC" + ) + """ + self.check_query_error(query, "downsampling.aggregation must be one of AVG, COUNT, DEFAULT_AGGREGATION, LAST, MAX, MIN, SUM, but has ABC") + + # check `downsampling.fill` setting validation + query = """ + SELECT * FROM local_monitoring.my_service WITH ( + selectors = @@{test_type="setting_validation"}@@, + `downsampling.fill` = "ABC" + ) + """ + self.check_query_error(query, "downsampling.fill must be one of NONE, NULL, PREVIOUS, but has ABC") + + # check `downsampling.grid_interval` setting validation + query = """ + SELECT * FROM local_monitoring.my_service WITH ( + selectors = @@{test_type="setting_validation"}@@, + `downsampling.grid_interval` = "ABC" + ) + """ + self.check_query_error(query, "downsampling.grid_interval must be positive number, but has ABC") + + # check unknown setting validation + query = """ + SELECT * FROM local_monitoring.my_service WITH ( + selectors = @@{test_type="setting_validation"}@@, + unk = "ABC" + ) + """ + self.check_query_error(query, "Unknown setting unk") + + # check additional downsampling settings validation + query = """ + SELECT * FROM local_monitoring.my_service WITH ( + selectors = @@{test_type="setting_validation"}@@, + `downsampling.disabled` = "true", + `downsampling.aggregation` = "AVG", + `downsampling.fill` = "PREVIOUS", + `downsampling.grid_interval` = "15" + ) + """ + self.check_query_error(query, "downsampling.disabled must be false if downsampling.aggregation, downsampling.fill or downsampling.grid_interval is specified") + + drop_source_query = """ + DROP EXTERNAL DATA SOURCE local_monitoring; + """ + result, error = self.execute_query(drop_source_query) + assert error is None + + @link_test_case("#23189") + def test_settings_validation_monitoring_program(self): + data_source_query = f""" + CREATE EXTERNAL DATA SOURCE local_monitoring WITH ( + SOURCE_TYPE = "Solomon", + LOCATION = "{self.solomon_http_endpoint}", + GRPC_LOCATION = "{self.solomon_grpc_endpoint}", + PROJECT = "settings_validation", + CLUSTER = "settings_validation", + AUTH_METHOD = "NONE", + USE_TLS = "false" + ) + """ + result, error = self.execute_query(data_source_query) + assert error is None + + query = """ + SELECT * FROM local_monitoring.my_service WITH ( + program = @@{test_type="setting_validation"}@@, + selectors = @@{test_type="setting_validation"}@@ + ) + """ + self.check_query_error(query, "either program or selectors must be specified") + + # check `labels` validation + query = """ + SELECT * FROM local_monitoring.my_service WITH ( + program = @@{test_type="setting_validation"}@@, + labels = "[test_type]" + ) + """ + self.check_query_error(query, "Label names should be specified in \"label1 [as alias1], label2 [as alias2], ...\" format") + + # check `labels` validation + query = """ + SELECT * FROM local_monitoring.my_service WITH ( + program = @@{test_type="setting_validation"}@@, + labels = @@"test_type"@@ + ) + """ + self.check_query_error(query, "Label names should be specified in \"label1 [as alias1], label2 [as alias2], ...\" format") + # check `from` setting validation query = """ SELECT * FROM local_monitoring.my_service WITH ( @@ -232,7 +494,7 @@ def test_settings_validation_monitoring(self): from = "invalid time" ) """ - self.check_query_error(query, "couldn\'t parse `from`, use Iso8601 format") + self.check_query_error(query, "couldn\'t parse `from`, use ISO8601 format") # check `to` setting validation query = """ @@ -241,7 +503,7 @@ def test_settings_validation_monitoring(self): to = "invalid time" ) """ - self.check_query_error(query, "couldn\'t parse `to`, use Iso8601 format") + self.check_query_error(query, "couldn\'t parse `to`, use ISO8601 format") # check `downsampling.disabled` setting validation query = """ @@ -299,3 +561,21 @@ def test_settings_validation_monitoring(self): ) """ self.check_query_error(query, "downsampling.disabled must be false if downsampling.aggregation, downsampling.fill or downsampling.grid_interval is specified") + + query = """ + SELECT * FROM local_monitoring.my_service WITH ( + selectors = @@{test_type="setting_validation"}@@, + + from = "1970-01-01T00:00:00Z", + to = "9999-01-01T00:00:01Z" + ) + """ + result, error = self.execute_query(query) + assert error is None + assert len(result[0].rows) == 0 + + drop_source_query = """ + DROP EXTERNAL DATA SOURCE local_monitoring; + """ + result, error = self.execute_query(drop_source_query) + assert error is None