Skip to content

Commit 4f3cbc9

Browse files
author
atarasov5
committed
YQL-20350: Fix singular types
commit_hash:b776960a1c683287ff0b5b23d95b2d48b0edccb5
1 parent 1b75a8b commit 4f3cbc9

19 files changed

+326
-54
lines changed

‎yql/essentials/minikql/comp_nodes/mkql_block_getelem.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ class TBlockGetElementExec {
156156
return arrow::Status::OK();
157157
}
158158
case EOptionalityHandlerStrategy::ReturnNull:
159-
*res = arrow::NullArray(tuple->length);
159+
*res = NYql::NUdf::MakeSingularArray(/*isNull=*/true, tuple->length);
160160
return arrow::Status::OK();
161161
}
162162

@@ -187,7 +187,7 @@ class TBlockGetElementExec {
187187
return arrow::Status::OK();
188188
}
189189
case EOptionalityHandlerStrategy::ReturnNull:
190-
*res = arrow::MakeNullScalar(ReturnArrowType);
190+
*res = NYql::NUdf::MakeSingularScalar(/*IsNull=*/true);
191191
return arrow::Status::OK();
192192
}
193193

‎yql/essentials/minikql/comp_nodes/ut/mkql_block_fuzzer.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,8 +212,10 @@ struct TFuzzerTraits {
212212
}
213213
}
214214

215+
template <bool IsNull>
215216
static std::unique_ptr<TResult> MakeSingular(const NYql::NUdf::TType* type,
216217
const TTypeEnvironment& env) {
218+
Y_UNUSED(IsNull);
217219
return std::make_unique<TLeafOffsetFuzzer</*IsOptional=*/false>>(type, env);
218220
}
219221
};

‎yql/essentials/minikql/computation/mkql_block_impl.cpp‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,7 @@ arrow::Datum DoConvertScalar(TType* type, const T& value, arrow::MemoryPool& poo
190190
}
191191

192192
if (IsSingularType(type)) {
193-
return arrow::MakeNullScalar(arrowType);
193+
return NYql::NUdf::MakeSingularScalar(type->IsNull());
194194
}
195195

196196
MKQL_ENSURE(false, "Unsupported type " << *type);
@@ -267,7 +267,9 @@ TBlockFuncNode::TBlockFuncNode(TComputationMutables& mutables,
267267
, StateIndex_(mutables.CurValueIndex++)
268268
, ArgsNodes_(std::move(argsNodes))
269269
, ArgsValuesDescr_(ToValueDescr(argsTypes))
270+
, ArgTypes_(argsTypes)
270271
, OutValueDescr_(ToValueDescr(outputType))
272+
, OutputType_(outputType)
271273
, Kernel_(kernel)
272274
, KernelHolder_(std::move(kernelHolder))
273275
, Options_(functionOptions)
@@ -283,7 +285,7 @@ NUdf::TUnboxedValuePod TBlockFuncNode::DoCalculate(TComputationContext& ctx) con
283285
for (ui32 i = 0; i < ArgsNodes_.size(); ++i) {
284286
const auto& value = ArgsNodes_[i]->GetValue(ctx);
285287
argDatums.emplace_back(TArrowBlock::From(value).GetDatum());
286-
ValidateDatum(argDatums.back(), ArgsValuesDescr_[i], ValidateDatumMode_);
288+
ValidateDatum(argDatums.back(), ArgsValuesDescr_[i], ArgTypes_[i], ValidateDatumMode_);
287289
}
288290

289291
if (ScalarOutput_) {
@@ -311,7 +313,7 @@ NUdf::TUnboxedValuePod TBlockFuncNode::DoCalculate(TComputationContext& ctx) con
311313
ForEachArrayData(output, [&](const auto& arr) { arrays.push_back(arr); });
312314
}
313315
auto resultArray = MakeArray(arrays);
314-
ValidateDatum(resultArray, OutValueDescr_, ValidateDatumMode_);
316+
ValidateDatum(resultArray, OutValueDescr_, OutputType_, ValidateDatumMode_);
315317
return ctx.HolderFactory.CreateArrowBlock(std::move(resultArray));
316318
}
317319

‎yql/essentials/minikql/computation/mkql_block_impl.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,9 @@ class TBlockFuncNode: public TMutableComputationNode<TBlockFuncNode> {
9090
const ui32 StateIndex_;
9191
const TComputationNodePtrVector ArgsNodes_;
9292
const std::vector<arrow::ValueDescr> ArgsValuesDescr_;
93+
const TVector<TType*> ArgTypes_;
9394
arrow::ValueDescr OutValueDescr_;
95+
const TType* const OutputType_ = nullptr;
9496
const arrow::compute::ScalarKernel& Kernel_;
9597
const std::shared_ptr<arrow::compute::ScalarKernel> KernelHolder_;
9698
const arrow::compute::FunctionOptions* const Options_;

‎yql/essentials/minikql/computation/mkql_block_reader.cpp‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
#include <yql/essentials/minikql/mkql_node_cast.h>
66

77
#include <yql/essentials/public/udf/udf_type_inspection.h>
8+
#include <yql/essentials/public/udf/udf_value_utils.h>
89

910
#include <arrow/array/array_binary.h>
1011
#include <arrow/chunked_array.h>
@@ -162,16 +163,17 @@ class TStringBlockItemConverter: public IBlockItemConverter {
162163
i32 TypeLen_ = 0;
163164
};
164165

166+
template <bool IsNull>
165167
class TSingularTypeItemConverter: public IBlockItemConverter {
166168
public:
167169
NUdf::TUnboxedValuePod MakeValue(TBlockItem item, const THolderFactory& holderFactory) const final {
168170
Y_UNUSED(item, holderFactory);
169-
return NUdf::TUnboxedValuePod::Zero();
171+
return NYql::NUdf::CreateSingularUnboxedValuePod<IsNull>();
170172
}
171173

172174
TBlockItem MakeItem(const NUdf::TUnboxedValuePod& value) const final {
173175
Y_UNUSED(value);
174-
return TBlockItem::Zero();
176+
return NYql::NUdf::CreateSingularBlockItem<IsNull>();
175177
}
176178
};
177179

@@ -303,7 +305,8 @@ struct TConverterTraits {
303305
using TExtOptional = TExternalOptionalBlockItemConverter;
304306
template <typename TTzDate, bool Nullable>
305307
using TTzDateConverter = TTzDateBlockItemConverter<TTzDate, Nullable>;
306-
using TSingularType = TSingularTypeItemConverter;
308+
template <bool IsNull>
309+
using TSingularType = TSingularTypeItemConverter<IsNull>;
307310

308311
constexpr static bool PassType = false;
309312

@@ -345,8 +348,9 @@ struct TConverterTraits {
345348
}
346349
}
347350

351+
template <bool IsNull>
348352
static std::unique_ptr<TResult> MakeSingular() {
349-
return std::make_unique<TSingularType>();
353+
return std::make_unique<TSingularType<IsNull>>();
350354
}
351355
};
352356

‎yql/essentials/minikql/computation/mkql_block_transport.cpp‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -586,6 +586,7 @@ class TExtOptionalBlockDeserializer final: public TBlockDeserializerBase {
586586
const std::unique_ptr<TBlockDeserializerBase> Inner_;
587587
};
588588

589+
template <bool IsNull>
589590
class TSingularTypeBlockSerializer final: public TBlockSerializerBase {
590591
using TBase = TBlockSerializerBase;
591592

@@ -608,6 +609,7 @@ class TSingularTypeBlockSerializer final: public TBlockSerializerBase {
608609
}
609610
};
610611

612+
template <bool IsNull>
611613
class TSingularTypeBlockDeserializer final: public TBlockDeserializerBase {
612614
using TBase = TBlockDeserializerBase;
613615

@@ -627,7 +629,7 @@ class TSingularTypeBlockDeserializer final: public TBlockDeserializerBase {
627629
Y_UNUSED(offset);
628630
Y_ENSURE(nullsCount == 0);
629631
Y_ENSURE(!nulls || nulls->size() == 0);
630-
return arrow::NullArray(blockLen).data();
632+
return NYql::NUdf::MakeSingularArray(IsNull, blockLen);
631633
}
632634

633635
std::shared_ptr<arrow::ArrayData> DoLoadArray(
@@ -639,7 +641,7 @@ class TSingularTypeBlockDeserializer final: public TBlockDeserializerBase {
639641
Y_UNUSED(offset, src);
640642
Y_ENSURE(nullsCount == 0);
641643
Y_ENSURE(!nulls || nulls->size() == 0);
642-
return arrow::NullArray(blockLen).data();
644+
return NYql::NUdf::MakeSingularArray(IsNull, blockLen);
643645
}
644646

645647
bool IsNullable() const final {
@@ -906,7 +908,8 @@ struct TSerializerTraits {
906908
using TExtOptional = TExtOptionalBlockSerializer;
907909
template <typename TTzDateType, bool Nullable>
908910
using TTzDate = TTzDateBlockSerializer<TTzDateType, Nullable>;
909-
using TSingularType = TSingularTypeBlockSerializer;
911+
template <bool IsNull>
912+
using TSingularType = TSingularTypeBlockSerializer<IsNull>;
910913
constexpr static bool PassType = false;
911914

912915
static std::unique_ptr<TResult> MakePg(
@@ -925,8 +928,9 @@ struct TSerializerTraits {
925928
ythrow yexception() << "Serializer not implemented for block resources";
926929
}
927930

931+
template <bool IsNull>
928932
static std::unique_ptr<TResult> MakeSingular(const TBlockSerializerParams& params) {
929-
return std::make_unique<TSingularType>(params);
933+
return std::make_unique<TSingularType<IsNull>>(params);
930934
}
931935

932936
template <typename TTzDateType>
@@ -950,7 +954,8 @@ struct TDeserializerTraits {
950954
using TExtOptional = TExtOptionalBlockDeserializer;
951955
template <typename TTzDateType, bool Nullable>
952956
using TTzDate = TTzDateBlockDeserializer<TTzDateType, Nullable>;
953-
using TSingularType = TSingularTypeBlockDeserializer;
957+
template <bool IsNull>
958+
using TSingularType = TSingularTypeBlockDeserializer<IsNull>;
954959

955960
constexpr static bool PassType = false;
956961

@@ -970,8 +975,9 @@ struct TDeserializerTraits {
970975
ythrow yexception() << "Deserializer not implemented for block resources";
971976
}
972977

978+
template <bool IsNull>
973979
static std::unique_ptr<TResult> MakeSingular(const TBlockSerializerParams& params) {
974-
return std::make_unique<TSingularType>(params);
980+
return std::make_unique<TSingularType<IsNull>>(params);
975981
}
976982

977983
template <typename TTzDateType>

‎yql/essentials/minikql/computation/mkql_block_trimmer.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -279,7 +279,9 @@ struct TTrimmerTraits {
279279
}
280280
}
281281

282+
template <bool IsNull>
282283
static TResult::TPtr MakeSingular(arrow::MemoryPool* pool) {
284+
Y_UNUSED(IsNull);
283285
return std::make_unique<TSingular>(pool);
284286
}
285287

‎yql/essentials/minikql/computation/mkql_computation_node_pack_ut.cpp‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -783,7 +783,7 @@ class TMiniKQLComputationNodePackTest: public TTestBase {
783783
TBlockItem tzDate{i};
784784
tzDate.SetTimezoneId(i % 100);
785785
builder4->Add(tzDate);
786-
builder5->Add(TBlockItem::Zero());
786+
builder5->Add(TBlockItem());
787787
switch (i % 3) {
788788
case 0:
789789
builder6->Add(TBlockItem(i));
@@ -933,7 +933,7 @@ class TMiniKQLComputationNodePackTest: public TTestBase {
933933
UNIT_ASSERT(b4.Get<ui16>() == i);
934934
UNIT_ASSERT(b4.GetTimezoneId() == (i % 100));
935935
TBlockItem b5 = reader5->GetItem(*TArrowBlock::From(unpackedColumns[legacyStruct ? 6 : 5]).GetDatum().array(), i - offset);
936-
UNIT_ASSERT(b5);
936+
UNIT_ASSERT(!b5);
937937
TBlockItem b6 = reader6->GetItem(*TArrowBlock::From(unpackedColumns[legacyStruct ? 7 : 6]).GetDatum().array(), i - offset);
938938
switch (i % 3) {
939939
case 0:
@@ -1127,9 +1127,9 @@ class TMiniKQLComputationNodePackTest: public TTestBase {
11271127
auto* rowType = PgmBuilder_.NewMultiType({blockNullType, scalarUi64Type});
11281128

11291129
auto builder = MakeArrayBuilder(TTypeInfoHelper(), nullType, *ArrowPool_, CalcBlockLen(CalcMaxBlockItemSize(nullType)), nullptr);
1130-
builder->Add(NYql::NUdf::TBlockItem::Zero());
1131-
builder->Add(NYql::NUdf::TBlockItem::Zero());
1132-
builder->Add(NYql::NUdf::TBlockItem::Zero());
1130+
builder->Add(NYql::NUdf::TBlockItem());
1131+
builder->Add(NYql::NUdf::TBlockItem());
1132+
builder->Add(NYql::NUdf::TBlockItem());
11331133
TUnboxedValueVector columns;
11341134
columns.emplace_back(HolderFactory_.CreateArrowBlock(builder->Build(/*finish=*/true)));
11351135
columns.emplace_back(HolderFactory_.CreateArrowBlock(arrow::Datum(std::make_shared<arrow::UInt64Scalar>(3))));
@@ -1156,9 +1156,9 @@ class TMiniKQLComputationNodePackTest: public TTestBase {
11561156
auto* rowType = PgmBuilder_.NewMultiType({blockOptNullType, scalarUi64Type});
11571157

11581158
auto builder = MakeArrayBuilder(TTypeInfoHelper(), optNullType, *ArrowPool_, CalcBlockLen(CalcMaxBlockItemSize(optNullType)), nullptr);
1159-
builder->Add(NYql::NUdf::TBlockItem::Zero());
1159+
builder->Add(NYql::NUdf::TBlockItem().MakeOptional());
11601160
builder->Add(NYql::NUdf::TBlockItem());
1161-
builder->Add(NYql::NUdf::TBlockItem::Zero());
1161+
builder->Add(NYql::NUdf::TBlockItem().MakeOptional());
11621162
TUnboxedValueVector columns;
11631163
columns.emplace_back(HolderFactory_.CreateArrowBlock(builder->Build(/*finish=*/true)));
11641164
columns.emplace_back(HolderFactory_.CreateArrowBlock(arrow::Datum(std::make_shared<arrow::UInt64Scalar>(3))));

0 commit comments

Comments
 (0)