Skip to content

Commit 2daef2e

Browse files
gridnevvvitgridnevvvit
committed
add validation for different precisions
1 parent 020b684 commit 2daef2e

File tree

4 files changed

+224
-2
lines changed

4 files changed

+224
-2
lines changed

‎ydb/core/kqp/ut/query/kqp_query_ut.cpp‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,146 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
7474
UNIT_ASSERT_VALUES_EQUAL(counters.RecompileRequestGet()->Val(), 1);
7575
}
7676

77+
Y_UNIT_TEST_TWIN(DecimalOutOfPrecisionBulk, EnableParameterizedDecimal) {
78+
TKikimrSettings serverSettings;
79+
serverSettings.FeatureFlags.SetEnableParameterizedDecimal(EnableParameterizedDecimal);
80+
serverSettings.WithSampleTables = false;
81+
82+
TKikimrRunner kikimr(serverSettings);
83+
auto client = kikimr.GetQueryClient();
84+
85+
{
86+
auto ddlResult = client.ExecuteQuery(R"(
87+
CREATE TABLE DecTest (
88+
Key Int32 NOT NULL,
89+
Value Decimal(22, 9) NOT NULL,
90+
PRIMARY KEY (Key)
91+
);
92+
)", NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync();
93+
UNIT_ASSERT_C(ddlResult.IsSuccess(), ddlResult.GetIssues().ToString());
94+
}
95+
96+
// 10000000000000 in Decimal(35, 9), invalid for Decimal(22, 9)
97+
Ydb::Value value;
98+
value.set_low_128(1864712049423024128);
99+
value.set_high_128(542);
100+
auto invalidValue = TDecimalValue(value, NYdb::TDecimalType(22, 9));
101+
102+
{
103+
auto db = kikimr.GetTableClient();
104+
auto session = db.CreateSession().GetValueSync().GetSession();
105+
106+
NYdb::TValueBuilder rows;
107+
rows.BeginList();
108+
rows.AddListItem()
109+
.BeginStruct()
110+
.AddMember("Key").Int32(1)
111+
.AddMember("Value").Decimal(TDecimalValue("10", 22, 9))
112+
.EndStruct();
113+
rows.AddListItem()
114+
.BeginStruct()
115+
.AddMember("Key").Int32(2)
116+
.AddMember("Value").Decimal(invalidValue)
117+
.EndStruct();
118+
rows.AddListItem()
119+
.BeginStruct()
120+
.AddMember("Key").Int32(3)
121+
.AddMember("Value").Decimal(TDecimalValue("10000000000000", 22, 9))
122+
.EndStruct();
123+
rows.EndList();
124+
125+
auto resultUpsert = db.BulkUpsert("/Root/DecTest", rows.Build()).GetValueSync();
126+
// TODO: Plan A, upsert should fail as provided value is invalid for given type.
127+
UNIT_ASSERT_C(!resultUpsert.IsSuccess(), resultUpsert.GetIssues().ToString());
128+
129+
auto tableYson = ReadTableToYson(session, "/Root/DecTest");
130+
// TODO: Plan B, value for key 2 should be inf, as provided value is out of range
131+
// for given type.
132+
CompareYson(R"([])", tableYson);
133+
}
134+
}
135+
136+
Y_UNIT_TEST_QUAD(DecimalOutOfPrecision, UseOltpSink, EnableParameterizedDecimal) {
137+
TKikimrSettings serverSettings;
138+
serverSettings.AppConfig.MutableTableServiceConfig()->SetEnableOltpSink(UseOltpSink);
139+
serverSettings.FeatureFlags.SetEnableParameterizedDecimal(EnableParameterizedDecimal);
140+
serverSettings.WithSampleTables = false;
141+
142+
TKikimrRunner kikimr(serverSettings);
143+
auto client = kikimr.GetQueryClient();
144+
145+
{
146+
auto ddlResult = client.ExecuteQuery(Sprintf(R"(
147+
CREATE TABLE DecTest (
148+
Key Int32 NOT NULL,
149+
Value Decimal(22, 9),
150+
%s
151+
PRIMARY KEY (Key)
152+
);
153+
)", EnableParameterizedDecimal ? " ValueLarge Decimal(35, 9), " : ""), NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync();
154+
UNIT_ASSERT_C(ddlResult.IsSuccess(), ddlResult.GetIssues().ToString());
155+
}
156+
157+
// 10000000000000 in Decimal(35, 9), invalid for Decimal(22, 9)
158+
Ydb::Value value;
159+
value.set_low_128(1864712049423024128);
160+
value.set_high_128(542);
161+
auto invalidValue = TDecimalValue(value, NYdb::TDecimalType(22, 9));
162+
163+
auto validValue = TDecimalValue(value, NYdb::TDecimalType(35, 9));
164+
165+
{
166+
auto params = TParamsBuilder()
167+
.AddParam("$value").Decimal(invalidValue).Build()
168+
.Build();
169+
170+
auto writeResult = client.ExecuteQuery(R"(
171+
UPSERT INTO DecTest (Key, Value) VALUES
172+
(1, CAST(10 AS Decimal(22,9))),
173+
(2, $value),
174+
(3, $value - CAST(1 AS Decimal(22,9)));
175+
)", NYdb::NQuery::TTxControl::BeginTx().CommitTx(), params).ExtractValueSync();
176+
177+
// TODO: Plan A, query should fail as provided value is invalid for given type.
178+
UNIT_ASSERT_C(!writeResult.IsSuccess(), writeResult.GetIssues().ToString());
179+
UNIT_ASSERT_EQUAL_C(writeResult.GetStatus(), EStatus::BAD_REQUEST, writeResult.GetIssues().ToString());
180+
181+
// TODO: Plan B, value for key 2 should be inf, as provided value is out of range
182+
// for given type.
183+
auto session = kikimr.GetTableClient().CreateSession().GetValueSync().GetSession();
184+
auto tableYson = ReadTableToYson(session, "/Root/DecTest");
185+
CompareYson(R"([])", tableYson);
186+
187+
if (EnableParameterizedDecimal)
188+
{
189+
auto paramsValid = TParamsBuilder()
190+
.AddParam("$value").Decimal(validValue).Build()
191+
.Build();
192+
auto writeResult = client.ExecuteQuery(R"(
193+
UPSERT INTO DecTest (Key, ValueLarge) VALUES
194+
(2, $value);
195+
)", NYdb::NQuery::TTxControl::BeginTx().CommitTx(), paramsValid).ExtractValueSync();
196+
197+
// TODO: Plan A, query should fail as provided value is invalid for given type.
198+
UNIT_ASSERT_C(writeResult.IsSuccess(), writeResult.GetIssues().ToString());
199+
UNIT_ASSERT_EQUAL_C(writeResult.GetStatus(), EStatus::SUCCESS, writeResult.GetIssues().ToString());
200+
201+
auto tableYson = ReadTableToYson(session, "/Root/DecTest");
202+
CompareYson(R"([[[2];#;["10000000000000"]]])", tableYson);
203+
}
204+
205+
if (EnableParameterizedDecimal)
206+
{
207+
auto writeResult = client.ExecuteQuery(R"(
208+
UPSERT INTO DecTest (Key, Value) SELECT Key, ValueLarge as Value FROM DecTest;
209+
)", NYdb::NQuery::TTxControl::BeginTx().CommitTx()).ExtractValueSync();
210+
// TODO: Plan A, query should fail as provided value is invalid for given type.
211+
UNIT_ASSERT_C(!writeResult.IsSuccess(), writeResult.GetIssues().ToString());
212+
UNIT_ASSERT_EQUAL_C(writeResult.GetStatus(), EStatus::GENERIC_ERROR, writeResult.GetIssues().ToString());
213+
}
214+
}
215+
}
216+
77217
Y_UNIT_TEST(QueryCache) {
78218
TKikimrRunner kikimr;
79219
auto db = kikimr.GetTableClient();

‎ydb/core/ydb_convert/ydb_convert.cpp‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
#include <ydb/library/ydb_issue/issue_helpers.h>
66
#include <ydb/core/protos/table_stats.pb.h>
77
#include <ydb/core/protos/subdomains.pb.h>
8+
#include <ydb/library/mkql_proto/mkql_proto.h>
89

910
#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/value/value.h>
1011

@@ -1278,7 +1279,21 @@ bool CellFromProtoVal(const NScheme::TTypeInfo& type, i32 typmod, const Ydb::Val
12781279
}
12791280
break;
12801281
}
1281-
case NScheme::NTypeIds::Decimal :
1282+
case NScheme::NTypeIds::Decimal : {
1283+
1284+
std::pair<ui64,ui64>& valInPool = *valueDataPool.Allocate<std::pair<ui64,ui64> >();
1285+
valInPool.first = val.low_128();
1286+
valInPool.second = val.high_128();
1287+
ui8 precision = type.GetDecimalType().GetPrecision();
1288+
auto validate = NYql::NDecimal::FromHalfs(val.low_128(), val.high_128());
1289+
if (!IsValidDecimal(precision, validate)) {
1290+
err = "Invalid decimal value";
1291+
return false;
1292+
}
1293+
1294+
c = TCell((const char*)&valInPool, sizeof(valInPool));
1295+
break;
1296+
}
12821297
case NScheme::NTypeIds::Uuid : {
12831298
std::pair<ui64,ui64>& valInPool = *valueDataPool.Allocate<std::pair<ui64,ui64> >();
12841299
valInPool.first = val.low_128();

‎ydb/library/mkql_proto/mkql_proto.cpp‎

Lines changed: 67 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,62 @@ namespace NKikimr::NMiniKQL {
2020

2121
namespace {
2222

23+
static constexpr std::array<std::pair<NYql::NDecimal::TInt128, NYql::NDecimal::TInt128>, NYql::NDecimal::MaxPrecision + 1> DecimalBounds = {
24+
NYql::NDecimal::GetBounds<0>(),
25+
NYql::NDecimal::GetBounds<1>(),
26+
NYql::NDecimal::GetBounds<2>(),
27+
NYql::NDecimal::GetBounds<3>(),
28+
NYql::NDecimal::GetBounds<4>(),
29+
NYql::NDecimal::GetBounds<5>(),
30+
NYql::NDecimal::GetBounds<6>(),
31+
NYql::NDecimal::GetBounds<7>(),
32+
NYql::NDecimal::GetBounds<8>(),
33+
NYql::NDecimal::GetBounds<9>(),
34+
NYql::NDecimal::GetBounds<10>(),
35+
NYql::NDecimal::GetBounds<11>(),
36+
NYql::NDecimal::GetBounds<12>(),
37+
NYql::NDecimal::GetBounds<13>(),
38+
NYql::NDecimal::GetBounds<14>(),
39+
NYql::NDecimal::GetBounds<15>(),
40+
NYql::NDecimal::GetBounds<16>(),
41+
NYql::NDecimal::GetBounds<17>(),
42+
NYql::NDecimal::GetBounds<18>(),
43+
NYql::NDecimal::GetBounds<19>(),
44+
NYql::NDecimal::GetBounds<20>(),
45+
NYql::NDecimal::GetBounds<21>(),
46+
NYql::NDecimal::GetBounds<22>(),
47+
NYql::NDecimal::GetBounds<23>(),
48+
NYql::NDecimal::GetBounds<24>(),
49+
NYql::NDecimal::GetBounds<25>(),
50+
NYql::NDecimal::GetBounds<26>(),
51+
NYql::NDecimal::GetBounds<27>(),
52+
NYql::NDecimal::GetBounds<28>(),
53+
NYql::NDecimal::GetBounds<29>(),
54+
NYql::NDecimal::GetBounds<30>(),
55+
NYql::NDecimal::GetBounds<31>(),
56+
NYql::NDecimal::GetBounds<32>(),
57+
NYql::NDecimal::GetBounds<33>(),
58+
NYql::NDecimal::GetBounds<34>(),
59+
NYql::NDecimal::GetBounds<35>(),
60+
};
61+
62+
bool IsValidDecimal(ui8 precision, NYql::NDecimal::TInt128 v) {
63+
if (NYql::NDecimal::IsError(v))
64+
return false;
65+
66+
if (NYql::NDecimal::IsNan(v))
67+
return true;
68+
69+
if (NYql::NDecimal::IsInf(v))
70+
return true;
71+
72+
if (precision >= DecimalBounds.size())
73+
return false;
74+
75+
const auto& db = DecimalBounds[precision];
76+
return v > db.first && v < db.second;
77+
}
78+
2379
void ExportTypeToProtoImpl(TType* type, NKikimrMiniKQL::TType& res, const TVector<ui32>* columnOrder = nullptr);
2480

2581
Y_FORCE_INLINE void HandleKindDataExport(const TType* type, const NUdf::TUnboxedValuePod& value, Ydb::Value& res) {
@@ -1628,7 +1684,17 @@ Y_FORCE_INLINE NUdf::TUnboxedValue KindDataImport(const TType* type, const Ydb::
16281684
return MakeString(value.bytes_value());
16291685
}
16301686
case NUdf::TDataType<NUdf::TDecimal>::Id: {
1631-
return NUdf::TUnboxedValuePod(NYql::NDecimal::FromHalfs(value.low_128(), value.high_128()));
1687+
auto data = NYql::NDecimal::FromHalfs(value.low_128(), value.high_128());
1688+
auto dataType = static_cast<const TDataType*>(type);
1689+
auto schemeType = dataType->GetSchemeType();
1690+
Y_ENSURE(schemeType == NYql::NProto::TypeIds::Decimal, "Expected decimal type, but found " << schemeType);
1691+
auto decimalType = static_cast<const TDataDecimalType *>(dataType);
1692+
auto params = decimalType->GetParams();
1693+
ui8 precision = params.first;
1694+
if (!IsValidDecimal(precision, data)) {
1695+
throw yexception() << "Invalid Decimal value for precision: " << precision;
1696+
}
1697+
return NUdf::TUnboxedValuePod(data);
16321698
}
16331699
default: {
16341700
throw yexception() << "Unsupported data type: " << dataType->GetSchemeType();

‎ydb/library/mkql_proto/mkql_proto.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ void ExportValueToProto(TType* type, const NUdf::TUnboxedValuePod& value, Ydb::V
2020

2121

2222
TType* ImportTypeFromProto(const NKikimrMiniKQL::TType& type, const TTypeEnvironment& env);
23+
bool IsValidDecimal(ui8 precision, NYql::NDecimal::TInt128 v);
2324

2425
std::pair<TType*, NUdf::TUnboxedValue> ImportValueFromProto(const Ydb::Type& type, const Ydb::Value& value,
2526
const TTypeEnvironment& env, const THolderFactory& factory);

0 commit comments

Comments
 (0)