Skip to content

Commit 6d0f0b2

Browse files
miloyipmiloyip
committed
Merge pull request Tencent#564 from corporateshark/stringnumbers
Implemented feature as in Tencent#560 (ints/doubles as strings)
2 parents 29c9580 + b5966c3 commit 6d0f0b2

File tree

10 files changed

+182
-48
lines changed

10 files changed

+182
-48
lines changed

‎example/capitalize/capitalize.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,8 @@ struct CapitalizeFilter {
2424
bool Int64(int64_t i) { return out_.Int64(i); }
2525
bool Uint64(uint64_t u) { return out_.Uint64(u); }
2626
bool Double(double d) { return out_.Double(d); }
27-
bool String(const char* str, SizeType length, bool) {
27+
bool RawNumber(const char* str, SizeType length, bool copy) { return out_.RawNumber(str, length, copy); }
28+
bool String(const char* str, SizeType length, bool) {
2829
buffer_.clear();
2930
for (SizeType i = 0; i < length; i++)
3031
buffer_.push_back(static_cast<char>(std::toupper(str[i])));

‎example/jsonx/jsonx.cpp

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,13 @@ class JsonxWriter {
5757
return WriteNumberElement(buffer, sprintf(buffer, "%.17g", d));
5858
}
5959

60+
bool RawNumber(const char* str, SizeType length, bool) {
61+
return
62+
WriteStartElement("number") &&
63+
WriteEscapedText(str, length) &&
64+
WriteEndElement("number");
65+
}
66+
6067
bool String(const char* str, SizeType length, bool) {
6168
return
6269
WriteStartElement("string") &&

‎example/simplereader/simplereader.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@ struct MyHandler {
1212
bool Int64(int64_t i) { cout << "Int64(" << i << ")" << endl; return true; }
1313
bool Uint64(uint64_t u) { cout << "Uint64(" << u << ")" << endl; return true; }
1414
bool Double(double d) { cout << "Double(" << d << ")" << endl; return true; }
15+
bool RawNumber(const char* str, SizeType length, bool copy) {
16+
cout << "Number(" << str << ", " << length << ", " << boolalpha << copy << ")" << endl;
17+
return true;
18+
}
1519
bool String(const char* str, SizeType length, bool copy) {
1620
cout << "String(" << str << ", " << length << ", " << boolalpha << copy << ")" << endl;
1721
return true;

‎include/rapidjson/document.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2325,6 +2325,14 @@ class GenericDocument : public GenericValue<Encoding, Allocator> {
23252325
bool Uint64(uint64_t i) { new (stack_.template Push<ValueType>()) ValueType(i); return true; }
23262326
bool Double(double d) { new (stack_.template Push<ValueType>()) ValueType(d); return true; }
23272327

2328+
bool RawNumber(const Ch* str, SizeType length, bool copy) {
2329+
if (copy)
2330+
new (stack_.template Push<ValueType>()) ValueType(str, length, GetAllocator());
2331+
else
2332+
new (stack_.template Push<ValueType>()) ValueType(str, length);
2333+
return true;
2334+
}
2335+
23282336
bool String(const Ch* str, SizeType length, bool copy) {
23292337
if (copy)
23302338
new (stack_.template Push<ValueType>()) ValueType(str, length, GetAllocator());

‎include/rapidjson/prettywriter.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,12 @@ class PrettyWriter : public Writer<OutputStream, SourceEncoding, TargetEncoding,
7474
bool Uint64(uint64_t u64) { PrettyPrefix(kNumberType); return Base::WriteUint64(u64); }
7575
bool Double(double d) { PrettyPrefix(kNumberType); return Base::WriteDouble(d); }
7676

77+
bool RawNumber(const Ch* str, SizeType length, bool copy = false) {
78+
(void)copy;
79+
PrettyPrefix(kNumberType);
80+
return Base::WriteString(str, length);
81+
}
82+
7783
bool String(const Ch* str, SizeType length, bool copy = false) {
7884
(void)copy;
7985
PrettyPrefix(kStringType);

‎include/rapidjson/reader.h

Lines changed: 71 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,7 @@ enum ParseFlag {
148148
kParseStopWhenDoneFlag = 8, //!< After parsing a complete JSON root from stream, stop further processing the rest of stream. When this flag is used, parser will not generate kParseErrorDocumentRootNotSingular error.
149149
kParseFullPrecisionFlag = 16, //!< Parse number in full precision (but slower).
150150
kParseCommentsFlag = 32, //!< Allow one-line (//) and multi-line (/**/) comments.
151+
kParseNumbersAsStringsFlag = 64, //!< Parse all numbers (ints/doubles) as strings.
151152
kParseDefaultFlags = RAPIDJSON_PARSE_DEFAULT_FLAGS //!< Default parse flags. Can be customized by defining RAPIDJSON_PARSE_DEFAULT_FLAGS
152153
};
153154

@@ -169,6 +170,8 @@ concept Handler {
169170
bool Int64(int64_t i);
170171
bool Uint64(uint64_t i);
171172
bool Double(double d);
173+
/// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length)
174+
bool RawNumber(const Ch* str, SizeType length, bool copy);
172175
bool String(const Ch* str, SizeType length, bool copy);
173176
bool StartObject();
174177
bool Key(const Ch* str, SizeType length, bool copy);
@@ -199,6 +202,8 @@ struct BaseReaderHandler {
199202
bool Int64(int64_t) { return static_cast<Override&>(*this).Default(); }
200203
bool Uint64(uint64_t) { return static_cast<Override&>(*this).Default(); }
201204
bool Double(double) { return static_cast<Override&>(*this).Default(); }
205+
/// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length)
206+
bool RawNumber(const Ch* str, SizeType len, bool copy) { return static_cast<Override&>(*this).String(str, len, copy); }
202207
bool String(const Ch*, SizeType, bool) { return static_cast<Override&>(*this).Default(); }
203208
bool StartObject() { return static_cast<Override&>(*this).Default(); }
204209
bool Key(const Ch* str, SizeType len, bool copy) { return static_cast<Override&>(*this).String(str, len, copy); }
@@ -1053,6 +1058,8 @@ class GenericReader {
10531058
RAPIDJSON_FORCEINLINE Ch Peek() const { return is.Peek(); }
10541059
RAPIDJSON_FORCEINLINE Ch TakePush() { return is.Take(); }
10551060
RAPIDJSON_FORCEINLINE Ch Take() { return is.Take(); }
1061+
RAPIDJSON_FORCEINLINE void Push(char) {}
1062+
10561063
size_t Tell() { return is.Tell(); }
10571064
size_t Length() { return 0; }
10581065
const char* Pop() { return 0; }
@@ -1075,6 +1082,10 @@ class GenericReader {
10751082
return Base::is.Take();
10761083
}
10771084

1085+
RAPIDJSON_FORCEINLINE void Push(char c) {
1086+
stackStream.Put(c);
1087+
}
1088+
10781089
size_t Length() { return stackStream.Length(); }
10791090

10801091
const char* Pop() {
@@ -1089,7 +1100,11 @@ class GenericReader {
10891100
template<unsigned parseFlags, typename InputStream, typename Handler>
10901101
void ParseNumber(InputStream& is, Handler& handler) {
10911102
internal::StreamLocalCopy<InputStream> copy(is);
1092-
NumberStream<InputStream, (parseFlags & kParseFullPrecisionFlag) != 0> s(*this, copy.s);
1103+
NumberStream<InputStream,
1104+
((parseFlags & kParseNumbersAsStringsFlag) != 0) ?
1105+
((parseFlags & kParseInsituFlag) == 0) :
1106+
((parseFlags & kParseFullPrecisionFlag) != 0)> s(*this, copy.s);
1107+
10931108
size_t startOffset = s.Tell();
10941109

10951110
// Parse minus
@@ -1176,6 +1191,9 @@ class GenericReader {
11761191
int expFrac = 0;
11771192
size_t decimalPosition;
11781193
if (Consume(s, '.')) {
1194+
if (((parseFlags & kParseNumbersAsStringsFlag) != 0) && ((parseFlags & kParseInsituFlag) == 0)) {
1195+
s.Push('.');
1196+
}
11791197
decimalPosition = s.Length();
11801198

11811199
if (RAPIDJSON_UNLIKELY(!(s.Peek() >= '0' && s.Peek() <= '9')))
@@ -1223,7 +1241,11 @@ class GenericReader {
12231241
// Parse exp = e [ minus / plus ] 1*DIGIT
12241242
int exp = 0;
12251243
if (Consume(s, 'e') || Consume(s, 'E')) {
1226-
if (!useDouble) {
1244+
if ( ((parseFlags & kParseNumbersAsStringsFlag) != 0) && ((parseFlags & kParseInsituFlag) == 0) ) {
1245+
s.Push( 'e' );
1246+
}
1247+
1248+
if (!useDouble) {
12271249
d = static_cast<double>(use64bit ? i64 : i);
12281250
useDouble = true;
12291251
}
@@ -1263,31 +1285,58 @@ class GenericReader {
12631285

12641286
// Finish parsing, call event according to the type of number.
12651287
bool cont = true;
1266-
size_t length = s.Length();
1267-
const char* decimal = s.Pop(); // Pop stack no matter if it will be used or not.
12681288

1269-
if (useDouble) {
1270-
int p = exp + expFrac;
1271-
if (parseFlags & kParseFullPrecisionFlag)
1272-
d = internal::StrtodFullPrecision(d, p, decimal, length, decimalPosition, exp);
1273-
else
1274-
d = internal::StrtodNormalPrecision(d, p);
1289+
if (parseFlags & kParseNumbersAsStringsFlag) {
12751290

1276-
cont = handler.Double(minus ? -d : d);
1277-
}
1278-
else {
1279-
if (use64bit) {
1280-
if (minus)
1281-
cont = handler.Int64(static_cast<int64_t>(~i64 + 1));
1282-
else
1283-
cont = handler.Uint64(i64);
1291+
if (parseFlags & kParseInsituFlag) {
1292+
s.Pop(); // Pop stack no matter if it will be used or not.
1293+
typename InputStream::Ch* head = is.PutBegin();
1294+
const size_t length = s.Tell() - startOffset;
1295+
RAPIDJSON_ASSERT(length <= 0xFFFFFFFF);
1296+
// *(head + length) = '\0';
1297+
const typename TargetEncoding::Ch* const str = reinterpret_cast<typename TargetEncoding::Ch*>(head);
1298+
cont = handler.RawNumber(str, SizeType(length), false);
12841299
}
12851300
else {
1286-
if (minus)
1287-
cont = handler.Int(static_cast<int32_t>(~i + 1));
1288-
else
1289-
cont = handler.Uint(i);
1301+
StackStream<typename TargetEncoding::Ch> stackStream(stack_);
1302+
SizeType numCharsToCopy = static_cast<SizeType>(s.Length());
1303+
while (numCharsToCopy--) {
1304+
Transcoder<SourceEncoding, TargetEncoding>::Transcode(is, stackStream);
1305+
}
1306+
stackStream.Put('\0');
1307+
const typename TargetEncoding::Ch* str = stackStream.Pop();
1308+
const SizeType length = static_cast<SizeType>(stackStream.Length()) - 1;
1309+
cont = handler.RawNumber(str, SizeType(length), true);
12901310
}
1311+
1312+
}
1313+
else {
1314+
size_t length = s.Length();
1315+
const char* decimal = s.Pop(); // Pop stack no matter if it will be used or not.
1316+
1317+
if (useDouble) {
1318+
int p = exp + expFrac;
1319+
if (parseFlags & kParseFullPrecisionFlag)
1320+
d = internal::StrtodFullPrecision(d, p, decimal, length, decimalPosition, exp);
1321+
else
1322+
d = internal::StrtodNormalPrecision(d, p);
1323+
1324+
cont = handler.Double(minus ? -d : d);
1325+
}
1326+
else {
1327+
if (use64bit) {
1328+
if (minus)
1329+
cont = handler.Int64(static_cast<int64_t>(~i64 + 1));
1330+
else
1331+
cont = handler.Uint64(i64);
1332+
}
1333+
else {
1334+
if (minus)
1335+
cont = handler.Int(static_cast<int32_t>(~i + 1));
1336+
else
1337+
cont = handler.Uint(i);
1338+
}
1339+
}
12911340
}
12921341
if (RAPIDJSON_UNLIKELY(!cont))
12931342
RAPIDJSON_PARSE_ERROR(kParseErrorTermination, startOffset);

‎include/rapidjson/schema.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,11 @@ class Hasher {
183183
return WriteNumber(n);
184184
}
185185

186+
bool RawNumber(const Ch* str, SizeType len, bool) {
187+
WriteBuffer(kNumberType, str, len * sizeof(Ch));
188+
return true;
189+
}
190+
186191
bool String(const Ch* str, SizeType len, bool) {
187192
WriteBuffer(kStringType, str, len * sizeof(Ch));
188193
return true;
@@ -1679,6 +1684,8 @@ RAPIDJSON_MULTILINEMACRO_END
16791684
bool Int64(int64_t i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int64, (CurrentContext(), i), (i)); }
16801685
bool Uint64(uint64_t u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint64, (CurrentContext(), u), (u)); }
16811686
bool Double(double d) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Double, (CurrentContext(), d), (d)); }
1687+
bool RawNumber(const Ch* str, SizeType length, bool copy)
1688+
{ RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); }
16821689
bool String(const Ch* str, SizeType length, bool copy)
16831690
{ RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); }
16841691

‎include/rapidjson/writer.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,12 @@ class Writer {
181181
*/
182182
bool Double(double d) { Prefix(kNumberType); return WriteDouble(d); }
183183

184+
bool RawNumber(const Ch* str, SizeType length, bool copy = false) {
185+
(void)copy;
186+
Prefix(kNumberType);
187+
return WriteString(str, length);
188+
}
189+
184190
bool String(const Ch* str, SizeType length, bool copy = false) {
185191
(void)copy;
186192
Prefix(kStringType);

‎test/unittest/readertest.cpp

Lines changed: 58 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1170,6 +1170,8 @@ struct IterativeParsingReaderHandler {
11701170

11711171
bool Double(double) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_DOUBLE; return true; }
11721172

1173+
bool RawNumber(const Ch*, SizeType, bool) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_STRING; return true; }
1174+
11731175
bool String(const Ch*, SizeType, bool) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_STRING; return true; }
11741176

11751177
bool StartObject() { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_STARTOBJECT; return true; }
@@ -1349,12 +1351,13 @@ struct TerminateHandler {
13491351
bool Int64(int64_t) { return e != 4; }
13501352
bool Uint64(uint64_t) { return e != 5; }
13511353
bool Double(double) { return e != 6; }
1352-
bool String(const char*, SizeType, bool) { return e != 7; }
1353-
bool StartObject() { return e != 8; }
1354-
bool Key(const char*, SizeType, bool) { return e != 9; }
1355-
bool EndObject(SizeType) { return e != 10; }
1356-
bool StartArray() { return e != 11; }
1357-
bool EndArray(SizeType) { return e != 12; }
1354+
bool RawNumber(const char*, SizeType, bool) { return e != 7; }
1355+
bool String(const char*, SizeType, bool) { return e != 8; }
1356+
bool StartObject() { return e != 9; }
1357+
bool Key(const char*, SizeType, bool) { return e != 10; }
1358+
bool EndObject(SizeType) { return e != 11; }
1359+
bool StartArray() { return e != 12; }
1360+
bool EndArray(SizeType) { return e != 13; }
13581361
};
13591362

13601363
#define TEST_TERMINATION(e, json)\
@@ -1375,14 +1378,15 @@ TEST(Reader, ParseTerminationByHandler) {
13751378
TEST_TERMINATION(4, "[-1234567890123456789");
13761379
TEST_TERMINATION(5, "[1234567890123456789");
13771380
TEST_TERMINATION(6, "[0.5]");
1378-
TEST_TERMINATION(7, "[\"a\"");
1379-
TEST_TERMINATION(8, "[{");
1380-
TEST_TERMINATION(9, "[{\"a\"");
1381-
TEST_TERMINATION(10, "[{}");
1382-
TEST_TERMINATION(10, "[{\"a\":1}"); // non-empty object
1383-
TEST_TERMINATION(11, "{\"a\":[");
1384-
TEST_TERMINATION(12, "{\"a\":[]");
1385-
TEST_TERMINATION(12, "{\"a\":[1]"); // non-empty array
1381+
// RawNumber() is never called
1382+
TEST_TERMINATION(8, "[\"a\"");
1383+
TEST_TERMINATION(9, "[{");
1384+
TEST_TERMINATION(10, "[{\"a\"");
1385+
TEST_TERMINATION(11, "[{}");
1386+
TEST_TERMINATION(11, "[{\"a\":1}"); // non-empty object
1387+
TEST_TERMINATION(12, "{\"a\":[");
1388+
TEST_TERMINATION(13, "{\"a\":[]");
1389+
TEST_TERMINATION(13, "{\"a\":[1]"); // non-empty array
13861390
}
13871391

13881392
TEST(Reader, ParseComments) {
@@ -1508,6 +1512,46 @@ TEST(Reader, UnrecognizedComment) {
15081512
EXPECT_EQ(kParseErrorUnspecificSyntaxError, reader.GetParseErrorCode());
15091513
}
15101514

1515+
struct NumbersAsStringsHandler {
1516+
bool Null() { return true; }
1517+
bool Bool(bool) { return true; }
1518+
bool Int(int) { return true; }
1519+
bool Uint(unsigned) { return true; }
1520+
bool Int64(int64_t) { return true; }
1521+
bool Uint64(uint64_t) { return true; }
1522+
bool Double(double) { return true; }
1523+
// 'str' is not null-terminated
1524+
bool RawNumber(const char* str, SizeType length, bool) {
1525+
EXPECT_TRUE(str != 0);
1526+
EXPECT_TRUE(strncmp(str, "3.1416", length) == 0);
1527+
return true;
1528+
}
1529+
bool String(const char*, SizeType, bool) { return true; }
1530+
bool StartObject() { return true; }
1531+
bool Key(const char*, SizeType, bool) { return true; }
1532+
bool EndObject(SizeType) { return true; }
1533+
bool StartArray() { return true; }
1534+
bool EndArray(SizeType) { return true; }
1535+
};
1536+
1537+
TEST(Reader, NumbersAsStrings) {
1538+
{
1539+
const char* json = "{ \"pi\": 3.1416 } ";
1540+
StringStream s(json);
1541+
NumbersAsStringsHandler h;
1542+
Reader reader;
1543+
EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
1544+
}
1545+
{
1546+
char* json = StrDup("{ \"pi\": 3.1416 } ");
1547+
InsituStringStream s(json);
1548+
NumbersAsStringsHandler h;
1549+
Reader reader;
1550+
EXPECT_TRUE(reader.Parse<kParseInsituFlag|kParseNumbersAsStringsFlag>(s, h));
1551+
free(json);
1552+
}
1553+
}
1554+
15111555
#ifdef __GNUC__
15121556
RAPIDJSON_DIAG_POP
15131557
#endif

‎test/unittest/valuetest.cpp

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1644,12 +1644,13 @@ struct TerminateHandler {
16441644
bool Int64(int64_t) { return e != 4; }
16451645
bool Uint64(uint64_t) { return e != 5; }
16461646
bool Double(double) { return e != 6; }
1647-
bool String(const char*, SizeType, bool) { return e != 7; }
1648-
bool StartObject() { return e != 8; }
1649-
bool Key(const char*, SizeType, bool) { return e != 9; }
1650-
bool EndObject(SizeType) { return e != 10; }
1651-
bool StartArray() { return e != 11; }
1652-
bool EndArray(SizeType) { return e != 12; }
1647+
bool RawNumber(const char*, SizeType, bool) { return e != 7; }
1648+
bool String(const char*, SizeType, bool) { return e != 8; }
1649+
bool StartObject() { return e != 9; }
1650+
bool Key(const char*, SizeType, bool) { return e != 10; }
1651+
bool EndObject(SizeType) { return e != 11; }
1652+
bool StartArray() { return e != 12; }
1653+
bool EndArray(SizeType) { return e != 13; }
16531654
};
16541655

16551656
#define TEST_TERMINATION(e, json)\
@@ -1670,12 +1671,13 @@ TEST(Value, AcceptTerminationByHandler) {
16701671
TEST_TERMINATION(4, "[-1234567890123456789]");
16711672
TEST_TERMINATION(5, "[9223372036854775808]");
16721673
TEST_TERMINATION(6, "[0.5]");
1673-
TEST_TERMINATION(7, "[\"a\"]");
1674-
TEST_TERMINATION(8, "[{}]");
1675-
TEST_TERMINATION(9, "[{\"a\":1}]");
1676-
TEST_TERMINATION(10, "[{}]");
1677-
TEST_TERMINATION(11, "{\"a\":[]}");
1674+
// RawNumber() is never called
1675+
TEST_TERMINATION(8, "[\"a\"]");
1676+
TEST_TERMINATION(9, "[{}]");
1677+
TEST_TERMINATION(10, "[{\"a\":1}]");
1678+
TEST_TERMINATION(11, "[{}]");
16781679
TEST_TERMINATION(12, "{\"a\":[]}");
1680+
TEST_TERMINATION(13, "{\"a\":[]}");
16791681
}
16801682

16811683
struct ValueIntComparer {

0 commit comments

Comments
 (0)