25-3-1: Make it possible to backup out-of-range decimal values (#26537)

Author: snaury

ydb/core/tx/datashard/type_serialization.cpp

@@ -15,7 +15,10 @@ TString DecimalToString(const std::pair<ui64, i64>& loHi, const NScheme::TTypeIn
     using namespace NYql::NDecimal;
 
     TInt128 val = FromHalfs(loHi.first, loHi.second);
-    return ToString(val, typeInfo.GetDecimalType().GetPrecision(), typeInfo.GetDecimalType().GetScale());
+    const char* result = ToString(val, MaxPrecision /*typeInfo.GetDecimalType().GetPrecision()*/, typeInfo.GetDecimalType().GetScale());
+    Y_ENSURE(result);
+
+    return result;
 }
 
 TString DyNumberToString(TStringBuf data) {
@@ -36,11 +39,15 @@ TString PgToString(TStringBuf data, const NScheme::TTypeInfo& typeInfo) {
 }
 
 bool DecimalToStream(const std::pair<ui64, i64>& loHi, IOutputStream& out, TString& err, const NScheme::TTypeInfo& typeInfo) {
-    Y_UNUSED(err);
     using namespace NYql::NDecimal;
 
     TInt128 val = FromHalfs(loHi.first, loHi.second);
-    out << ToString(val, typeInfo.GetDecimalType().GetPrecision(), typeInfo.GetDecimalType().GetScale());
+    const char* result = ToString(val, MaxPrecision /*typeInfo.GetDecimalType().GetPrecision()*/, typeInfo.GetDecimalType().GetScale());
+    if (!result) [[unlikely]] {
+        err = "Invalid Decimal binary representation";
+        return false;
+    }
+    out << result;
     return true;
 }
 

ydb/core/tx/schemeshard/ut_export/ut_export.cpp

@@ -2969,4 +2969,124 @@ attributes {
              NLs::IndexState(NKikimrSchemeOp::EIndexStateReady),
              NLs::IndexKeys({"value"})});
     }
+
+    Y_UNIT_TEST(DecimalOutOfRange) {
+        EnvOptions().DisableStatsBatching(true);
+        Env(); // Init test env
+        ui64 txId = 100;
+
+        TestCreateTable(Runtime(), ++txId, "/MyRoot", R"(
+                Name: "Table1"
+                Columns { Name: "key" Type: "Uint64" }
+                Columns { Name: "value" Type: "Decimal" }
+                KeyColumnNames: ["key"]
+            )");
+        Env().TestWaitNotification(Runtime(), txId);
+
+        // Write a normal decimal value
+        // 10.0^13-1 (scale 9) = 0x21e19e0c9ba76a53600
+        {
+            ui64 key = 1u;
+            std::pair<ui64, i64> value = { 0x19e0c9ba76a53600ULL, 0x21eULL };
+            UploadRow(Runtime(), "/MyRoot/Table1", 0, {1}, {2}, {TCell::Make(key)}, {TCell::Make(value)});
+        }
+        // Write a decimal value that is out of range for precision 22
+        // 10.0^13 (scale 9) = 10^22 = 0x21e19e0c9bab2400000
+        {
+            ui64 key = 2u;
+            std::pair<ui64, i64> value = { 0x19e0c9bab2400000ULL, 0x21eULL };
+            UploadRow(Runtime(), "/MyRoot/Table1", 0, {1}, {2}, {TCell::Make(key)}, {TCell::Make(value)});
+        }
+
+        TestExport(Runtime(), ++txId, "/MyRoot", Sprintf(R"(
+            ExportToS3Settings {
+              endpoint: "localhost:%d"
+              scheme: HTTP
+              items {
+                source_path: "/MyRoot/Table1"
+                destination_prefix: "Backup1"
+              }
+            }
+        )", S3Port()));
+        Env().TestWaitNotification(Runtime(), txId);
+
+        TestGetExport(Runtime(), txId, "/MyRoot", Ydb::StatusIds::SUCCESS);
+
+        UNIT_ASSERT(HasS3File("/Backup1/metadata.json"));
+        UNIT_ASSERT(HasS3File("/Backup1/data_00.csv"));
+        UNIT_ASSERT_STRINGS_EQUAL(GetS3FileContent("/Backup1/data_00.csv"),
+            "1,9999999999999\n"
+            "2,10000000000000\n");
+
+        TestImport(Runtime(), ++txId, "/MyRoot", Sprintf(R"(
+            ImportFromS3Settings {
+              endpoint: "localhost:%d"
+              scheme: HTTP
+              items {
+                source_prefix: "Backup1"
+                destination_path: "/MyRoot/Table2"
+              }
+            }
+        )", S3Port()));
+        Env().TestWaitNotification(Runtime(), txId);
+
+        TestGetImport(Runtime(), txId, "/MyRoot", Ydb::StatusIds::SUCCESS);
+
+        TestExport(Runtime(), ++txId, "/MyRoot", Sprintf(R"(
+            ExportToS3Settings {
+              endpoint: "localhost:%d"
+              scheme: HTTP
+              items {
+                source_path: "/MyRoot/Table2"
+                destination_prefix: "Backup2"
+              }
+            }
+        )", S3Port()));
+        Env().TestWaitNotification(Runtime(), txId);
+
+        TestGetExport(Runtime(), txId, "/MyRoot", Ydb::StatusIds::SUCCESS);
+
+        // Note: out-of-range values are restored as inf
+        UNIT_ASSERT(HasS3File("/Backup2/metadata.json"));
+        UNIT_ASSERT(HasS3File("/Backup2/data_00.csv"));
+        UNIT_ASSERT_STRINGS_EQUAL(GetS3FileContent("/Backup2/data_00.csv"),
+            "1,9999999999999\n"
+            "2,inf\n");
+    }
+
+    Y_UNIT_TEST(CorruptedDecimalValue) {
+        EnvOptions().DisableStatsBatching(true);
+        Env(); // Init test env
+        ui64 txId = 100;
+
+        TestCreateTable(Runtime(), ++txId, "/MyRoot", R"(
+                Name: "Table1"
+                Columns { Name: "key" Type: "Uint64" }
+                Columns { Name: "value" Type: "Decimal" }
+                KeyColumnNames: ["key"]
+            )");
+        Env().TestWaitNotification(Runtime(), txId);
+
+        // Write a decimal value that is way out of range for max precision 35
+        // 10^38 = 0x4b3b4ca85a86c47a098a224000000000
+        {
+            ui64 key = 1u;
+            std::pair<ui64, i64> value = { 0x098a224000000000ULL, 0x4b3b4ca85a86c47aULL };
+            UploadRow(Runtime(), "/MyRoot/Table1", 0, {1}, {2}, {TCell::Make(key)}, {TCell::Make(value)});
+        }
+
+        TestExport(Runtime(), ++txId, "/MyRoot", Sprintf(R"(
+            ExportToS3Settings {
+              endpoint: "localhost:%d"
+              scheme: HTTP
+              items {
+                source_path: "/MyRoot/Table1"
+                destination_prefix: "Backup1"
+              }
+            }
+        )", S3Port()));
+        Env().TestWaitNotification(Runtime(), txId);
+
+        TestGetExport(Runtime(), txId, "/MyRoot", Ydb::StatusIds::CANCELLED);
+    }
 }