Merge pull request Tencent#1290 from abolz/fix-strtod

Fix strtod

Author: miloyip

include/rapidjson/internal/biginteger.h

@@ -133,7 +133,7 @@ class BigInteger {
         RAPIDJSON_ASSERT(count_ + offset <= kCapacity);
 
         if (interShift == 0) {
-            std::memmove(&digits_[count_ - 1 + offset], &digits_[count_ - 1], count_ * sizeof(Type));
+            std::memmove(digits_ + offset, digits_, count_ * sizeof(Type));
             count_ += offset;
         }
         else {

include/rapidjson/internal/diyfp.h

@@ -1,15 +1,15 @@
 // Tencent is pleased to support the open source community by making RapidJSON available.
-// 
+//
 // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
 //
 // Licensed under the MIT License (the "License"); you may not use this file except
 // in compliance with the License. You may obtain a copy of the License at
 //
 // http://opensource.org/licenses/MIT
 //
-// Unless required by applicable law or agreed to in writing, software distributed 
-// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 
-// CONDITIONS OF ANY KIND, either express or implied. See the License for the 
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
 // specific language governing permissions and limitations under the License.
 
 // This is a C++ header-only implementation of Grisu2 algorithm from the publication:
@@ -20,6 +20,7 @@
 #define RAPIDJSON_DIYFP_H_
 
 #include "../rapidjson.h"
+#include <limits>
 
 #if defined(_MSC_VER) && defined(_M_AMD64) && !defined(__INTEL_COMPILER)
 #include <intrin.h>
@@ -56,7 +57,7 @@ struct DiyFp {
         if (biased_e != 0) {
             f = significand + kDpHiddenBit;
             e = biased_e - kDpExponentBias;
-        } 
+        }
         else {
             f = significand;
             e = kDpMinExponent + 1;
@@ -141,7 +142,16 @@ struct DiyFp {
             double d;
             uint64_t u64;
         }u;
-        const uint64_t be = (e == kDpDenormalExponent && (f & kDpHiddenBit) == 0) ? 0 : 
+        RAPIDJSON_ASSERT(f <= kDpHiddenBit + kDpSignificandMask);
+        if (e < kDpDenormalExponent) {
+            // Underflow.
+            return 0.0;
+        }
+        if (e >= kDpMaxExponent) {
+            // Overflow.
+            return std::numeric_limits<double>::infinity();
+        }
+        const uint64_t be = (e == kDpDenormalExponent && (f & kDpHiddenBit) == 0) ? 0 :
             static_cast<uint64_t>(e + kDpExponentBias);
         u.u64 = (f & kDpSignificandMask) | (be << kDpSignificandSize);
         return u.d;
@@ -220,9 +230,10 @@ inline DiyFp GetCachedPowerByIndex(size_t index) {
         641,   667,   694,   720,   747,   774,   800,   827,   853,   880,
         907,   933,   960,   986,  1013,  1039,  1066
     };
+    RAPIDJSON_ASSERT(index < 87);
     return DiyFp(kCachedPowers_F[index], kCachedPowers_E[index]);
 }
-    
+
 inline DiyFp GetCachedPower(int e, int* K) {
 
     //int k = static_cast<int>(ceil((-61 - e) * 0.30102999566398114)) + 374;
@@ -238,10 +249,11 @@ inline DiyFp GetCachedPower(int e, int* K) {
 }
 
 inline DiyFp GetCachedPower10(int exp, int *outExp) {
-     unsigned index = (static_cast<unsigned>(exp) + 348u) / 8u;
-     *outExp = -348 + static_cast<int>(index) * 8;
-     return GetCachedPowerByIndex(index);
- }
+    RAPIDJSON_ASSERT(exp >= -348);
+    unsigned index = static_cast<unsigned>(exp + 348) / 8u;
+    *outExp = -348 + static_cast<int>(index) * 8;
+    return GetCachedPowerByIndex(index);
+}
 
 #ifdef __GNUC__
 RAPIDJSON_DIAG_POP

include/rapidjson/internal/strtod.h

@@ -19,6 +19,8 @@
 #include "biginteger.h"
 #include "diyfp.h"
 #include "pow10.h"
+#include <climits>
+#include <limits>
 
 RAPIDJSON_NAMESPACE_BEGIN
 namespace internal {
@@ -126,20 +128,20 @@ inline bool StrtodFast(double d, int p, double* result) {
 }
 
 // Compute an approximation and see if it is within 1/2 ULP
-inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosition, int exp, double* result) {
+inline bool StrtodDiyFp(const char* decimals, int dLen, int dExp, double* result) {
     uint64_t significand = 0;
-    size_t i = 0;   // 2^64 - 1 = 18446744073709551615, 1844674407370955161 = 0x1999999999999999    
-    for (; i < length; i++) {
+    int i = 0;   // 2^64 - 1 = 18446744073709551615, 1844674407370955161 = 0x1999999999999999    
+    for (; i < dLen; i++) {
         if (significand  >  RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) ||
             (significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > '5'))
             break;
         significand = significand * 10u + static_cast<unsigned>(decimals[i] - '0');
     }
 
-    if (i < length && decimals[i] >= '5') // Rounding
+    if (i < dLen && decimals[i] >= '5') // Rounding
         significand++;
 
-    size_t remaining = length - i;
+    int remaining = dLen - i;
     const int kUlpShift = 3;
     const int kUlp = 1 << kUlpShift;
     int64_t error = (remaining == 0) ? 0 : kUlp / 2;
@@ -148,24 +150,24 @@ inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosit
     v = v.Normalize();
     error <<= -v.e;
 
-    const int dExp = static_cast<int>(decimalPosition) - static_cast<int>(i) + exp;
+    dExp += remaining;
 
     int actualExp;
     DiyFp cachedPower = GetCachedPower10(dExp, &actualExp);
     if (actualExp != dExp) {
         static const DiyFp kPow10[] = {
-            DiyFp(RAPIDJSON_UINT64_C2(0xa0000000, 00000000), -60),  // 10^1
-            DiyFp(RAPIDJSON_UINT64_C2(0xc8000000, 00000000), -57),  // 10^2
-            DiyFp(RAPIDJSON_UINT64_C2(0xfa000000, 00000000), -54),  // 10^3
-            DiyFp(RAPIDJSON_UINT64_C2(0x9c400000, 00000000), -50),  // 10^4
-            DiyFp(RAPIDJSON_UINT64_C2(0xc3500000, 00000000), -47),  // 10^5
-            DiyFp(RAPIDJSON_UINT64_C2(0xf4240000, 00000000), -44),  // 10^6
-            DiyFp(RAPIDJSON_UINT64_C2(0x98968000, 00000000), -40)   // 10^7
+            DiyFp(RAPIDJSON_UINT64_C2(0xa0000000, 0x00000000), -60),  // 10^1
+            DiyFp(RAPIDJSON_UINT64_C2(0xc8000000, 0x00000000), -57),  // 10^2
+            DiyFp(RAPIDJSON_UINT64_C2(0xfa000000, 0x00000000), -54),  // 10^3
+            DiyFp(RAPIDJSON_UINT64_C2(0x9c400000, 0x00000000), -50),  // 10^4
+            DiyFp(RAPIDJSON_UINT64_C2(0xc3500000, 0x00000000), -47),  // 10^5
+            DiyFp(RAPIDJSON_UINT64_C2(0xf4240000, 0x00000000), -44),  // 10^6
+            DiyFp(RAPIDJSON_UINT64_C2(0x98968000, 0x00000000), -40)   // 10^7
         };
-        int  adjustment = dExp - actualExp - 1;
-        RAPIDJSON_ASSERT(adjustment >= 0 && adjustment < 7);
-        v = v * kPow10[adjustment];
-        if (length + static_cast<unsigned>(adjustment)> 19u) // has more digits than decimal digits in 64-bit
+        int adjustment = dExp - actualExp;
+        RAPIDJSON_ASSERT(adjustment >= 1 && adjustment < 8);
+        v = v * kPow10[adjustment - 1];
+        if (dLen + adjustment > 19) // has more digits than decimal digits in 64-bit
             error += kUlp / 2;
     }
 
@@ -203,9 +205,9 @@ inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosit
     return halfWay - static_cast<unsigned>(error) >= precisionBits || precisionBits >= halfWay + static_cast<unsigned>(error);
 }
 
-inline double StrtodBigInteger(double approx, const char* decimals, size_t length, size_t decimalPosition, int exp) {
-    const BigInteger dInt(decimals, length);
-    const int dExp = static_cast<int>(decimalPosition) - static_cast<int>(length) + exp;
+inline double StrtodBigInteger(double approx, const char* decimals, int dLen, int dExp) {
+    RAPIDJSON_ASSERT(dLen >= 0);
+    const BigInteger dInt(decimals, static_cast<unsigned>(dLen));
     Double a(approx);
     int cmp = CheckWithinHalfULP(a.Value(), dInt, dExp);
     if (cmp < 0)
@@ -225,42 +227,61 @@ inline double StrtodFullPrecision(double d, int p, const char* decimals, size_t
     RAPIDJSON_ASSERT(d >= 0.0);
     RAPIDJSON_ASSERT(length >= 1);
 
-    double result;
+    double result = 0.0;
     if (StrtodFast(d, p, &result))
         return result;
 
+    RAPIDJSON_ASSERT(length <= INT_MAX);
+    int dLen = static_cast<int>(length);
+
+    RAPIDJSON_ASSERT(length >= decimalPosition);
+    RAPIDJSON_ASSERT(length - decimalPosition <= INT_MAX);
+    int dExpAdjust = static_cast<int>(length - decimalPosition);
+
+    RAPIDJSON_ASSERT(exp >= INT_MIN + dExpAdjust);
+    int dExp = exp - dExpAdjust;
+
+    // Make sure length+dExp does not overflow
+    RAPIDJSON_ASSERT(dExp <= INT_MAX - dLen);
+
     // Trim leading zeros
-    while (*decimals == '0' && length > 1) {
-        length--;
+    while (dLen > 0 && *decimals == '0') {
+        dLen--;
         decimals++;
-        decimalPosition--;
     }
 
     // Trim trailing zeros
-    while (decimals[length - 1] == '0' && length > 1) {
-        length--;
-        decimalPosition--;
-        exp++;
+    while (dLen > 0 && decimals[dLen - 1] == '0') {
+        dLen--;
+        dExp++;
+    }
+
+    if (dLen == 0) { // Buffer only contains zeros.
+        return 0.0;
     }
 
     // Trim right-most digits
-    const int kMaxDecimalDigit = 780;
-    if (static_cast<int>(length) > kMaxDecimalDigit) {
-        int delta = (static_cast<int>(length) - kMaxDecimalDigit);
-        exp += delta;
-        decimalPosition -= static_cast<unsigned>(delta);
-        length = kMaxDecimalDigit;
+    const int kMaxDecimalDigit = 767 + 1;
+    if (dLen > kMaxDecimalDigit) {
+        dExp += dLen - kMaxDecimalDigit;
+        dLen = kMaxDecimalDigit;
     }
 
-    // If too small, underflow to zero
-    if (int(length) + exp < -324)
+    // If too small, underflow to zero.
+    // Any x <= 10^-324 is interpreted as zero.
+    if (dLen + dExp <= -324)
         return 0.0;
 
-    if (StrtodDiyFp(decimals, length, decimalPosition, exp, &result))
+    // If too large, overflow to infinity.
+    // Any x >= 10^309 is interpreted as +infinity.
+    if (dLen + dExp > 309)
+        return std::numeric_limits<double>::infinity();
+
+    if (StrtodDiyFp(decimals, dLen, dExp, &result))
         return result;
 
     // Use approximation from StrtodDiyFp and make adjustment with BigInteger comparison
-    return StrtodBigInteger(result, decimals, length, decimalPosition, exp);
+    return StrtodBigInteger(result, decimals, dLen, dExp);
 }
 
 } // namespace internal

include/rapidjson/reader.h

@@ -1561,8 +1561,6 @@ class GenericReader {
         // Force double for big integer
         if (useDouble) {
             while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) {
-                if (RAPIDJSON_UNLIKELY(d >= 1.7976931348623157e307)) // DBL_MAX / 10.0
-                    RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, startOffset);
                 d = d * 10 + (s.TakePush() - '0');
             }
         }
@@ -1702,6 +1700,13 @@ class GenericReader {
                else
                    d = internal::StrtodNormalPrecision(d, p);
 
+               // Use > max, instead of == inf, to fix bogus warning -Wfloat-equal
+               if (d > std::numeric_limits<double>::max()) {
+                   // Overflow
+                   // TODO: internal::StrtodX should report overflow (or underflow)
+                   RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, startOffset);
+               }
+
                cont = handler.Double(minus ? -d : d);
            }
            else if (useNanOrInf) {

test/unittest/bigintegertest.cpp

@@ -120,6 +120,11 @@ TEST(BigInteger, LeftShift) {
     EXPECT_TRUE(BIGINTEGER_LITERAL("4537899042132549697536") == a);
     a <<= 99;
     EXPECT_TRUE(BIGINTEGER_LITERAL("2876235222267216943024851750785644982682875244576768") == a);
+
+    a = 1;
+    a <<= 64; // a.count_ != 1
+    a <<= 256; // interShift == 0
+    EXPECT_TRUE(BIGINTEGER_LITERAL("2135987035920910082395021706169552114602704522356652769947041607822219725780640550022962086936576") == a);
 }
 
 TEST(BigInteger, Compare) {