Skip to content

Commit 00dbcf2

Browse files
miloyipmiloyip
authored
Merge pull request Tencent#1926 from Kyrega/master
Fix RawNumber for characters with sizeof(Ch) > sizeof(char)
2 parents 28c59ab + 22ee8b0 commit 00dbcf2

File tree

4 files changed

+180
-30
lines changed

4 files changed

+180
-30
lines changed

‎include/rapidjson/internal/biginteger.h

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,8 @@ class BigInteger {
3737
digits_[0] = u;
3838
}
3939

40-
BigInteger(const char* decimals, size_t length) : count_(1) {
40+
template<typename Ch>
41+
BigInteger(const Ch* decimals, size_t length) : count_(1) {
4142
RAPIDJSON_ASSERT(length > 0);
4243
digits_[0] = 0;
4344
size_t i = 0;
@@ -221,7 +222,8 @@ class BigInteger {
221222
bool IsZero() const { return count_ == 1 && digits_[0] == 0; }
222223

223224
private:
224-
void AppendDecimal64(const char* begin, const char* end) {
225+
template<typename Ch>
226+
void AppendDecimal64(const Ch* begin, const Ch* end) {
225227
uint64_t u = ParseUint64(begin, end);
226228
if (IsZero())
227229
*this = u;
@@ -236,11 +238,12 @@ class BigInteger {
236238
digits_[count_++] = digit;
237239
}
238240

239-
static uint64_t ParseUint64(const char* begin, const char* end) {
241+
template<typename Ch>
242+
static uint64_t ParseUint64(const Ch* begin, const Ch* end) {
240243
uint64_t r = 0;
241-
for (const char* p = begin; p != end; ++p) {
242-
RAPIDJSON_ASSERT(*p >= '0' && *p <= '9');
243-
r = r * 10u + static_cast<unsigned>(*p - '0');
244+
for (const Ch* p = begin; p != end; ++p) {
245+
RAPIDJSON_ASSERT(*p >= Ch('0') && *p <= Ch('9'));
246+
r = r * 10u + static_cast<unsigned>(*p - Ch('0'));
244247
}
245248
return r;
246249
}

‎include/rapidjson/internal/strtod.h

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -128,17 +128,18 @@ inline bool StrtodFast(double d, int p, double* result) {
128128
}
129129

130130
// Compute an approximation and see if it is within 1/2 ULP
131-
inline bool StrtodDiyFp(const char* decimals, int dLen, int dExp, double* result) {
131+
template<typename Ch>
132+
inline bool StrtodDiyFp(const Ch* decimals, int dLen, int dExp, double* result) {
132133
uint64_t significand = 0;
133134
int i = 0; // 2^64 - 1 = 18446744073709551615, 1844674407370955161 = 0x1999999999999999
134135
for (; i < dLen; i++) {
135136
if (significand > RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) ||
136-
(significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > '5'))
137+
(significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > Ch('5')))
137138
break;
138-
significand = significand * 10u + static_cast<unsigned>(decimals[i] - '0');
139+
significand = significand * 10u + static_cast<unsigned>(decimals[i] - Ch('0'));
139140
}
140141

141-
if (i < dLen && decimals[i] >= '5') // Rounding
142+
if (i < dLen && decimals[i] >= Ch('5')) // Rounding
142143
significand++;
143144

144145
int remaining = dLen - i;
@@ -205,7 +206,8 @@ inline bool StrtodDiyFp(const char* decimals, int dLen, int dExp, double* result
205206
return halfWay - static_cast<unsigned>(error) >= precisionBits || precisionBits >= halfWay + static_cast<unsigned>(error);
206207
}
207208

208-
inline double StrtodBigInteger(double approx, const char* decimals, int dLen, int dExp) {
209+
template<typename Ch>
210+
inline double StrtodBigInteger(double approx, const Ch* decimals, int dLen, int dExp) {
209211
RAPIDJSON_ASSERT(dLen >= 0);
210212
const BigInteger dInt(decimals, static_cast<unsigned>(dLen));
211213
Double a(approx);
@@ -223,7 +225,8 @@ inline double StrtodBigInteger(double approx, const char* decimals, int dLen, in
223225
return a.NextPositiveDouble();
224226
}
225227

226-
inline double StrtodFullPrecision(double d, int p, const char* decimals, size_t length, size_t decimalPosition, int exp) {
228+
template<typename Ch>
229+
inline double StrtodFullPrecision(double d, int p, const Ch* decimals, size_t length, size_t decimalPosition, int exp) {
227230
RAPIDJSON_ASSERT(d >= 0.0);
228231
RAPIDJSON_ASSERT(length >= 1);
229232

‎include/rapidjson/reader.h

Lines changed: 20 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1404,11 +1404,11 @@ class GenericReader {
14041404
}
14051405
#endif // RAPIDJSON_NEON
14061406

1407-
template<typename InputStream, bool backup, bool pushOnTake>
1407+
template<typename InputStream, typename StackCharacter, bool backup, bool pushOnTake>
14081408
class NumberStream;
14091409

1410-
template<typename InputStream>
1411-
class NumberStream<InputStream, false, false> {
1410+
template<typename InputStream, typename StackCharacter>
1411+
class NumberStream<InputStream, StackCharacter, false, false> {
14121412
public:
14131413
typedef typename InputStream::Ch Ch;
14141414

@@ -1421,43 +1421,43 @@ class GenericReader {
14211421

14221422
size_t Tell() { return is.Tell(); }
14231423
size_t Length() { return 0; }
1424-
const char* Pop() { return 0; }
1424+
const StackCharacter* Pop() { return 0; }
14251425

14261426
protected:
14271427
NumberStream& operator=(const NumberStream&);
14281428

14291429
InputStream& is;
14301430
};
14311431

1432-
template<typename InputStream>
1433-
class NumberStream<InputStream, true, false> : public NumberStream<InputStream, false, false> {
1434-
typedef NumberStream<InputStream, false, false> Base;
1432+
template<typename InputStream, typename StackCharacter>
1433+
class NumberStream<InputStream, StackCharacter, true, false> : public NumberStream<InputStream, StackCharacter, false, false> {
1434+
typedef NumberStream<InputStream, StackCharacter, false, false> Base;
14351435
public:
14361436
NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is), stackStream(reader.stack_) {}
14371437

14381438
RAPIDJSON_FORCEINLINE Ch TakePush() {
1439-
stackStream.Put(static_cast<char>(Base::is.Peek()));
1439+
stackStream.Put(static_cast<StackCharacter>(Base::is.Peek()));
14401440
return Base::is.Take();
14411441
}
14421442

1443-
RAPIDJSON_FORCEINLINE void Push(char c) {
1443+
RAPIDJSON_FORCEINLINE void Push(StackCharacter c) {
14441444
stackStream.Put(c);
14451445
}
14461446

14471447
size_t Length() { return stackStream.Length(); }
14481448

1449-
const char* Pop() {
1449+
const StackCharacter* Pop() {
14501450
stackStream.Put('\0');
14511451
return stackStream.Pop();
14521452
}
14531453

14541454
private:
1455-
StackStream<char> stackStream;
1455+
StackStream<StackCharacter> stackStream;
14561456
};
14571457

1458-
template<typename InputStream>
1459-
class NumberStream<InputStream, true, true> : public NumberStream<InputStream, true, false> {
1460-
typedef NumberStream<InputStream, true, false> Base;
1458+
template<typename InputStream, typename StackCharacter>
1459+
class NumberStream<InputStream, StackCharacter, true, true> : public NumberStream<InputStream, StackCharacter, true, false> {
1460+
typedef NumberStream<InputStream, StackCharacter, true, false> Base;
14611461
public:
14621462
NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is) {}
14631463

@@ -1466,8 +1466,10 @@ class GenericReader {
14661466

14671467
template<unsigned parseFlags, typename InputStream, typename Handler>
14681468
void ParseNumber(InputStream& is, Handler& handler) {
1469+
typedef typename internal::SelectIf<internal::BoolType<(parseFlags & kParseNumbersAsStringsFlag) != 0>, typename TargetEncoding::Ch, char>::Type NumberCharacter;
1470+
14691471
internal::StreamLocalCopy<InputStream> copy(is);
1470-
NumberStream<InputStream,
1472+
NumberStream<InputStream, NumberCharacter,
14711473
((parseFlags & kParseNumbersAsStringsFlag) != 0) ?
14721474
((parseFlags & kParseInsituFlag) == 0) :
14731475
((parseFlags & kParseFullPrecisionFlag) != 0),
@@ -1692,10 +1694,10 @@ class GenericReader {
16921694
}
16931695
else {
16941696
SizeType numCharsToCopy = static_cast<SizeType>(s.Length());
1695-
StringStream srcStream(s.Pop());
1697+
GenericStringStream<UTF8<NumberCharacter>> srcStream(s.Pop());
16961698
StackStream<typename TargetEncoding::Ch> dstStream(stack_);
16971699
while (numCharsToCopy--) {
1698-
Transcoder<UTF8<>, TargetEncoding>::Transcode(srcStream, dstStream);
1700+
Transcoder<UTF8<typename TargetEncoding::Ch>, TargetEncoding>::Transcode(srcStream, dstStream);
16991701
}
17001702
dstStream.Put('\0');
17011703
const typename TargetEncoding::Ch* str = dstStream.Pop();
@@ -1705,7 +1707,7 @@ class GenericReader {
17051707
}
17061708
else {
17071709
size_t length = s.Length();
1708-
const char* decimal = s.Pop(); // Pop stack no matter if it will be used or not.
1710+
const NumberCharacter* decimal = s.Pop(); // Pop stack no matter if it will be used or not.
17091711

17101712
if (useDouble) {
17111713
int p = exp + expFrac;

‎test/unittest/readertest.cpp

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1392,6 +1392,36 @@ class IStreamWrapper {
13921392
std::istream& is_;
13931393
};
13941394

1395+
class WIStreamWrapper {
1396+
public:
1397+
typedef wchar_t Ch;
1398+
1399+
WIStreamWrapper(std::wistream& is) : is_(is) {}
1400+
1401+
Ch Peek() const {
1402+
unsigned c = is_.peek();
1403+
return c == std::char_traits<wchar_t>::eof() ? Ch('\0') : static_cast<Ch>(c);
1404+
}
1405+
1406+
Ch Take() {
1407+
unsigned c = is_.get();
1408+
return c == std::char_traits<wchar_t>::eof() ? Ch('\0') : static_cast<Ch>(c);
1409+
}
1410+
1411+
size_t Tell() const { return static_cast<size_t>(is_.tellg()); }
1412+
1413+
Ch* PutBegin() { assert(false); return 0; }
1414+
void Put(Ch) { assert(false); }
1415+
void Flush() { assert(false); }
1416+
size_t PutEnd(Ch*) { assert(false); return 0; }
1417+
1418+
private:
1419+
WIStreamWrapper(const WIStreamWrapper&);
1420+
WIStreamWrapper& operator=(const WIStreamWrapper&);
1421+
1422+
std::wistream& is_;
1423+
};
1424+
13951425
TEST(Reader, Parse_IStreamWrapper_StringStream) {
13961426
const char* json = "[1,2,3,4]";
13971427

@@ -1991,6 +2021,118 @@ TEST(Reader, NumbersAsStrings) {
19912021
}
19922022
}
19932023

2024+
struct NumbersAsStringsHandlerWChar_t {
2025+
bool Null() { return true; }
2026+
bool Bool(bool) { return true; }
2027+
bool Int(int) { return true; }
2028+
bool Uint(unsigned) { return true; }
2029+
bool Int64(int64_t) { return true; }
2030+
bool Uint64(uint64_t) { return true; }
2031+
bool Double(double) { return true; }
2032+
// 'str' is not null-terminated
2033+
bool RawNumber(const wchar_t* str, SizeType length, bool) {
2034+
EXPECT_TRUE(str != 0);
2035+
EXPECT_TRUE(expected_len_ == length);
2036+
EXPECT_TRUE(wcsncmp(str, expected_, length) == 0);
2037+
return true;
2038+
}
2039+
bool String(const wchar_t*, SizeType, bool) { return true; }
2040+
bool StartObject() { return true; }
2041+
bool Key(const wchar_t*, SizeType, bool) { return true; }
2042+
bool EndObject(SizeType) { return true; }
2043+
bool StartArray() { return true; }
2044+
bool EndArray(SizeType) { return true; }
2045+
2046+
NumbersAsStringsHandlerWChar_t(const wchar_t* expected)
2047+
: expected_(expected)
2048+
, expected_len_(wcslen(expected)) {}
2049+
2050+
const wchar_t* expected_;
2051+
size_t expected_len_;
2052+
};
2053+
2054+
TEST(Reader, NumbersAsStringsWChar_t) {
2055+
{
2056+
const wchar_t* json = L"{ \"pi\": 3.1416 } ";
2057+
GenericStringStream<UTF16<>> s(json);
2058+
NumbersAsStringsHandlerWChar_t h(L"3.1416");
2059+
GenericReader<UTF16<>, UTF16<>> reader;
2060+
EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
2061+
}
2062+
{
2063+
wchar_t* json = StrDup(L"{ \"pi\": 3.1416 } ");
2064+
GenericInsituStringStream<UTF16<>> s(json);
2065+
NumbersAsStringsHandlerWChar_t h(L"3.1416");
2066+
GenericReader<UTF16<>, UTF16<>> reader;
2067+
EXPECT_TRUE(reader.Parse<kParseInsituFlag | kParseNumbersAsStringsFlag>(s, h));
2068+
free(json);
2069+
}
2070+
{
2071+
const wchar_t* json = L"{ \"gigabyte\": 1.0e9 } ";
2072+
GenericStringStream<UTF16<>> s(json);
2073+
NumbersAsStringsHandlerWChar_t h(L"1.0e9");
2074+
GenericReader<UTF16<>, UTF16<>> reader;
2075+
EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
2076+
}
2077+
{
2078+
wchar_t* json = StrDup(L"{ \"gigabyte\": 1.0e9 } ");
2079+
GenericInsituStringStream<UTF16<>> s(json);
2080+
NumbersAsStringsHandlerWChar_t h(L"1.0e9");
2081+
GenericReader<UTF16<>, UTF16<>> reader;
2082+
EXPECT_TRUE(reader.Parse<kParseInsituFlag | kParseNumbersAsStringsFlag>(s, h));
2083+
free(json);
2084+
}
2085+
{
2086+
const wchar_t* json = L"{ \"pi\": 314.159e-2 } ";
2087+
GenericStringStream<UTF16<>> s(json);
2088+
NumbersAsStringsHandlerWChar_t h(L"314.159e-2");
2089+
GenericReader<UTF16<>, UTF16<>> reader;
2090+
EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
2091+
}
2092+
{
2093+
wchar_t* json = StrDup(L"{ \"gigabyte\": 314.159e-2 } ");
2094+
GenericInsituStringStream<UTF16<>> s(json);
2095+
NumbersAsStringsHandlerWChar_t h(L"314.159e-2");
2096+
GenericReader<UTF16<>, UTF16<>> reader;
2097+
EXPECT_TRUE(reader.Parse<kParseInsituFlag | kParseNumbersAsStringsFlag>(s, h));
2098+
free(json);
2099+
}
2100+
{
2101+
const wchar_t* json = L"{ \"negative\": -1.54321 } ";
2102+
GenericStringStream<UTF16<>> s(json);
2103+
NumbersAsStringsHandlerWChar_t h(L"-1.54321");
2104+
GenericReader<UTF16<>, UTF16<>> reader;
2105+
EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
2106+
}
2107+
{
2108+
wchar_t* json = StrDup(L"{ \"negative\": -1.54321 } ");
2109+
GenericInsituStringStream<UTF16<>> s(json);
2110+
NumbersAsStringsHandlerWChar_t h(L"-1.54321");
2111+
GenericReader<UTF16<>, UTF16<>> reader;
2112+
EXPECT_TRUE(reader.Parse<kParseInsituFlag | kParseNumbersAsStringsFlag>(s, h));
2113+
free(json);
2114+
}
2115+
{
2116+
const wchar_t* json = L"{ \"pi\": 314.159e-2 } ";
2117+
std::wstringstream ss(json);
2118+
WIStreamWrapper s(ss);
2119+
NumbersAsStringsHandlerWChar_t h(L"314.159e-2");
2120+
GenericReader<UTF16<>, UTF16<>> reader;
2121+
EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
2122+
}
2123+
{
2124+
wchar_t n1e319[321]; // '1' followed by 319 '0'
2125+
n1e319[0] = L'1';
2126+
for(int i = 1; i < 320; i++)
2127+
n1e319[i] = L'0';
2128+
n1e319[320] = L'\0';
2129+
GenericStringStream<UTF16<>> s(n1e319);
2130+
NumbersAsStringsHandlerWChar_t h(n1e319);
2131+
GenericReader<UTF16<>, UTF16<>> reader;
2132+
EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
2133+
}
2134+
}
2135+
19942136
template <unsigned extraFlags>
19952137
void TestTrailingCommas() {
19962138
{

0 commit comments

Comments
 (0)