Remove decimal overflow validator entirely

Decimal validation occurs on parse, when the input value (string, etc) is converted to an arrow::DecimalXX value.

Author: alexbaden

omniscidb/DataMgr/Encoder.cpp

@@ -118,7 +118,6 @@ Encoder* Encoder::Create(Data_Namespace::AbstractBuffer* buffer,
 Encoder::Encoder(Data_Namespace::AbstractBuffer* buffer)
     : num_elems_(0)
     , buffer_(buffer)
-    , decimal_overflow_validator_(buffer ? buffer->type() : nullptr)
     , date_days_overflow_validator_(buffer ? buffer->type() : nullptr){};
 
 std::shared_ptr<ChunkMetadata> Encoder::getMetadata() {

omniscidb/DataMgr/Encoder.h

@@ -36,61 +36,6 @@ class AbstractBuffer;
 // default max input buffer size to 1MB
 #define MAX_INPUT_BUF_SIZE 1048576
 
-class DecimalOverflowValidator {
- public:
-  DecimalOverflowValidator(const hdk::ir::Type* type) {
-    if (type && type->isArray()) {
-      type = type->as<hdk::ir::ArrayBaseType>()->elemType();
-    }
-
-    if (type && type->isDecimal()) {
-      do_check_ = true;
-      int precision = type->as<hdk::ir::DecimalType>()->precision();
-      int scale = type->as<hdk::ir::DecimalType>()->scale();
-      max_ = (int64_t)std::pow((double)10.0, precision);
-      min_ = -max_;
-      pow10_ = precision - scale;
-
-    } else {
-      do_check_ = false;
-      max_ = 1;
-      min_ = -1;
-      pow10_ = 0;
-    }
-  }
-
-  template <typename T>
-  void validate(T value) const {
-    if (std::is_integral<T>::value) {
-      do_validate(static_cast<int64_t>(value));
-    }
-  }
-
-  void do_validate(int64_t value) const {
-    if (!do_check_) {
-      return;
-    }
-
-    if (value >= max_) {
-      throw std::runtime_error("Decimal overflow: value is greater than 10^" +
-                               std::to_string(pow10_) + " max " + std::to_string(max_) +
-                               " value " + std::to_string(value));
-    }
-
-    if (value <= min_) {
-      throw std::runtime_error("Decimal overflow: value is less than -10^" +
-                               std::to_string(pow10_) + " min " + std::to_string(min_) +
-                               " value " + std::to_string(value));
-    }
-  }
-
- private:
-  bool do_check_;
-  int64_t max_;
-  int64_t min_;
-  int pow10_;
-};
-
 template <typename INNER_VALIDATOR>
 class NullAwareValidator {
  public:
@@ -253,7 +198,6 @@ class Encoder {
 
   Data_Namespace::AbstractBuffer* buffer_;
 
-  DecimalOverflowValidator decimal_overflow_validator_;
   DateDaysOverflowValidator date_days_overflow_validator_;
 };
 

omniscidb/DataMgr/FixedLengthArrayNoneEncoder.h

@@ -325,11 +325,9 @@ class FixedLengthArrayNoneEncoder : public Encoder {
               if (bigint_array[i] == NULL_BIGINT) {
                 has_nulls = true;
               } else if (initialized) {
-                decimal_overflow_validator_.validate(bigint_array[i]);
                 elem_min.bigintval = std::min(elem_min.bigintval, bigint_array[i]);
                 elem_max.bigintval = std::max(elem_max.bigintval, bigint_array[i]);
               } else {
-                decimal_overflow_validator_.validate(bigint_array[i]);
                 elem_min.bigintval = bigint_array[i];
                 elem_max.bigintval = bigint_array[i];
                 initialized = true;

omniscidb/DataMgr/FixedLengthEncoder.h

@@ -91,7 +91,6 @@ class FixedLengthEncoder : public Encoder {
           for (size_t i = range.begin(); i < range.end(); i++) {
             if (data[i] != inline_null_value<V>()) {
               if (!fixlen_array || data[i] != inline_null_array_value<V>()) {
-                decimal_overflow_validator_.validate(data[i]);
                 min = std::min(min, data[i]);
                 max = std::max(max, data[i]);
               }
@@ -189,7 +188,6 @@ class FixedLengthEncoder : public Encoder {
   V encodeDataAndUpdateStats(const T& unencoded_data) {
     V encoded_data = static_cast<V>(unencoded_data);
     if (unencoded_data != encoded_data) {
-      decimal_overflow_validator_.validate(unencoded_data);
       LOG(ERROR) << "Fixed encoding failed, Unencoded: " +
                         std::to_string(unencoded_data) +
                         " encoded: " + std::to_string(encoded_data);
@@ -198,7 +196,6 @@ class FixedLengthEncoder : public Encoder {
       if (data == std::numeric_limits<V>::min()) {
         has_nulls = true;
       } else {
-        decimal_overflow_validator_.validate(data);
         dataMin = std::min(dataMin, data);
         dataMax = std::max(dataMax, data);
       }

omniscidb/DataMgr/NoneEncoder.h

@@ -94,7 +94,6 @@ class NoneEncoder : public Encoder {
           for (size_t i = range.begin(); i < range.end(); i++) {
             if (data[i] != inline_null_value<T>()) {
               if (!fixlen_array || data[i] != inline_null_array_value<T>()) {
-                decimal_overflow_validator_.validate(data[i]);
                 min = std::min(min, data[i]);
                 max = std::max(max, data[i]);
               }
@@ -192,7 +191,6 @@ class NoneEncoder : public Encoder {
     if (unencoded_data == none_encoded_null_value<T>()) {
       has_nulls = true;
     } else {
-      decimal_overflow_validator_.validate(unencoded_data);
       dataMin = std::min(dataMin, unencoded_data);
       dataMax = std::max(dataMax, unencoded_data);
     }

omniscidb/Tests/ArrowStorageTest.cpp

@@ -1734,6 +1734,16 @@ TEST_F(ArrowStorageTest, AppendCsvData_Decimals) {
             std::vector<int64_t>({22200, 22222, 20000, inline_null_value<int64_t>()}));
 }
 
+TEST_F(ArrowStorageTest, AppendCsvData_Decimals_OutOfRange) {
+  ArrowStorage storage(TEST_SCHEMA_ID, "test", TEST_DB_ID, config_);
+  ArrowStorage::CsvParseOptions parse_options;
+  parse_options.header = false;
+  TableInfoPtr tinfo = storage.createTable(
+      "table1", {{"d1", ctx.decimal64(10, 4)}, {"d2", ctx.decimal64(10, 6)}});
+  EXPECT_ANY_THROW(storage.appendCsvData(
+      "12345678910.12345,12345678910.1234567", tinfo->table_id, parse_options));
+}
+
 TEST_F(ArrowStorageTest, AppendJsonData_DecimalArrays) {
   ArrowStorage storage(TEST_SCHEMA_ID, "test", TEST_DB_ID, config_);
   auto decimal_3 = ctx.arrayFixed(3, ctx.decimal64(10, 2));