Skip to content

Commit 91397f9

Browse files
bblanchonbblanchon
committed
Optimize storage of tiny strings (up to 3 characters)
1 parent 7f75985 commit 91397f9

File tree

16 files changed

+228
-56
lines changed

16 files changed

+228
-56
lines changed

‎CHANGELOG.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
ArduinoJson: change log
22
=======================
33

4+
HEAD
5+
----
6+
7+
* Optimize storage of tiny strings (up to 3 characters)
8+
49
v7.3.1 (2025-02-27)
510
------
611

‎extras/tests/JsonDeserializer/object.cpp‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -292,22 +292,23 @@ TEST_CASE("deserialize JSON object") {
292292
}
293293

294294
SECTION("Repeated key") {
295-
DeserializationError err = deserializeJson(doc, "{a:{b:{c:1}},a:2}");
295+
DeserializationError err =
296+
deserializeJson(doc, "{alfa:{bravo:{charlie:1}},alfa:2}");
296297

297298
REQUIRE(err == DeserializationError::Ok);
298-
REQUIRE(doc.as<std::string>() == "{\"a\":2}");
299+
REQUIRE(doc.as<std::string>() == "{\"alfa\":2}");
299300
REQUIRE(spy.log() ==
300301
AllocatorLog{
301302
Allocate(sizeofStringBuffer()),
302303
Allocate(sizeofPool()),
303-
Reallocate(sizeofStringBuffer(), sizeofString("a")),
304+
Reallocate(sizeofStringBuffer(), sizeofString("alfa")),
304305
Allocate(sizeofStringBuffer()),
305-
Reallocate(sizeofStringBuffer(), sizeofString("b")),
306+
Reallocate(sizeofStringBuffer(), sizeofString("bravo")),
306307
Allocate(sizeofStringBuffer()),
307-
Reallocate(sizeofStringBuffer(), sizeofString("c")),
308+
Reallocate(sizeofStringBuffer(), sizeofString("charlie")),
308309
Allocate(sizeofStringBuffer()),
309-
Deallocate(sizeofString("b")),
310-
Deallocate(sizeofString("c")),
310+
Deallocate(sizeofString("bravo")),
311+
Deallocate(sizeofString("charlie")),
311312
Deallocate(sizeofStringBuffer()),
312313
Reallocate(sizeofPool(), sizeofObject(2) + sizeofObject(1)),
313314
});
@@ -389,11 +390,11 @@ TEST_CASE("deserialize JSON object under memory constraints") {
389390

390391
SECTION("string allocation fails") {
391392
timebomb.setCountdown(3);
392-
char input[] = "{\"a\":\"b\"}";
393+
char input[] = "{\"alfa\":\"bravo\"}";
393394

394395
DeserializationError err = deserializeJson(doc, input);
395396

396397
REQUIRE(err == DeserializationError::NoMemory);
397-
REQUIRE(doc.as<std::string>() == "{\"a\":null}");
398+
REQUIRE(doc.as<std::string>() == "{\"alfa\":null}");
398399
}
399400
}

‎extras/tests/JsonObject/set.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -100,13 +100,13 @@ TEST_CASE("JsonObject::set()") {
100100
JsonDocument doc3(&timebomb);
101101
JsonObject obj3 = doc3.to<JsonObject>();
102102

103-
obj1["a"_s] = 1;
104-
obj1["b"_s] = 2;
103+
obj1["alpha"_s] = 1;
104+
obj1["beta"_s] = 2;
105105

106106
bool success = obj3.set(obj1);
107107

108108
REQUIRE(success == false);
109-
REQUIRE(doc3.as<std::string>() == "{\"a\":1}");
109+
REQUIRE(doc3.as<std::string>() == "{\"alpha\":1}");
110110
}
111111

112112
SECTION("copy fails in the middle of an array") {

‎extras/tests/JsonVariant/as.cpp‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,7 @@ TEST_CASE("JsonVariant::as()") {
199199
REQUIRE(variant.as<JsonString>() == "hello");
200200
}
201201

202-
SECTION("set(std::string(\"4.2\"))") {
202+
SECTION("set(std::string(\"4.2\")) (tiny string optimization)") {
203203
variant.set("4.2"_s);
204204

205205
REQUIRE(variant.as<bool>() == true);
@@ -211,6 +211,18 @@ TEST_CASE("JsonVariant::as()") {
211211
REQUIRE(variant.as<JsonString>().isStatic() == false);
212212
}
213213

214+
SECTION("set(std::string(\"123.45\"))") {
215+
variant.set("123.45"_s);
216+
217+
REQUIRE(variant.as<bool>() == true);
218+
REQUIRE(variant.as<long>() == 123L);
219+
REQUIRE(variant.as<double>() == Approx(123.45));
220+
REQUIRE(variant.as<const char*>() == "123.45"_s);
221+
REQUIRE(variant.as<std::string>() == "123.45"_s);
222+
REQUIRE(variant.as<JsonString>() == "123.45");
223+
REQUIRE(variant.as<JsonString>().isStatic() == false);
224+
}
225+
214226
SECTION("set(\"true\")") {
215227
variant.set("true");
216228

‎extras/tests/JsonVariant/set.cpp‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,18 @@ TEST_CASE("JsonVariant::set() when there is enough memory") {
6363
});
6464
}
6565

66+
SECTION("char* (tiny string optimization)") {
67+
char str[16];
68+
69+
strcpy(str, "abc");
70+
bool result = variant.set(str);
71+
strcpy(str, "def");
72+
73+
REQUIRE(result == true);
74+
REQUIRE(variant == "abc"); // stores by copy
75+
REQUIRE(spy.log() == AllocatorLog{});
76+
}
77+
6678
SECTION("(char*)0") {
6779
bool result = variant.set(static_cast<char*>(0));
6880

‎extras/tests/MsgPackDeserializer/deserializeArray.cpp‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,11 @@
55
#include <ArduinoJson.h>
66
#include <catch.hpp>
77

8+
#include "Allocators.hpp"
9+
810
TEST_CASE("deserialize MsgPack array") {
9-
JsonDocument doc;
11+
SpyingAllocator spy;
12+
JsonDocument doc(&spy);
1013

1114
SECTION("fixarray") {
1215
SECTION("empty") {
@@ -30,6 +33,24 @@ TEST_CASE("deserialize MsgPack array") {
3033
REQUIRE(array[0] == 1);
3134
REQUIRE(array[1] == 2);
3235
}
36+
37+
SECTION("tiny strings") {
38+
DeserializationError error =
39+
deserializeMsgPack(doc, "\x92\xA3xxx\xA3yyy");
40+
41+
REQUIRE(error == DeserializationError::Ok);
42+
REQUIRE(doc.is<JsonArray>());
43+
REQUIRE(doc.size() == 2);
44+
REQUIRE(doc[0] == "xxx");
45+
REQUIRE(doc[1] == "yyy");
46+
REQUIRE(spy.log() == AllocatorLog{
47+
Allocate(sizeofPool()),
48+
Allocate(sizeofString("xxx")),
49+
// Buffer is reused for the next string
50+
Deallocate(sizeofString("xxx")),
51+
Reallocate(sizeofPool(), sizeofPool(2)),
52+
});
53+
}
3354
}
3455

3556
SECTION("array 16") {

‎extras/tests/MsgPackDeserializer/deserializeVariant.cpp‎

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -348,28 +348,31 @@ TEST_CASE("deserializeMsgPack() under memory constaints") {
348348
SECTION("{}") {
349349
checkError(0, "\x80", DeserializationError::Ok);
350350
}
351-
SECTION("{H:1}") {
352-
checkError(1, "\x81\xA1H\x01", DeserializationError::NoMemory);
353-
checkError(2, "\x81\xA1H\x01", DeserializationError::Ok);
351+
SECTION("{Hello:1}") {
352+
checkError(1, "\x81\xA5Hello\x01", DeserializationError::NoMemory);
353+
checkError(2, "\x81\xA5Hello\x01", DeserializationError::Ok);
354354
}
355-
SECTION("{H:1,W:2}") {
356-
checkError(2, "\x82\xA1H\x01\xA1W\x02", DeserializationError::NoMemory);
357-
checkError(3, "\x82\xA1H\x01\xA1W\x02", DeserializationError::Ok);
355+
SECTION("{Hello:1,World:2}") {
356+
checkError(2, "\x82\xA5Hello\x01\xA5World\x02",
357+
DeserializationError::NoMemory);
358+
checkError(3, "\x82\xA5Hello\x01\xA5World\x02", DeserializationError::Ok);
358359
}
359360
}
360361

361362
SECTION("map 16") {
362363
SECTION("{}") {
363364
checkError(0, "\xDE\x00\x00", DeserializationError::Ok);
364365
}
365-
SECTION("{H:1}") {
366-
checkError(1, "\xDE\x00\x01\xA1H\x01", DeserializationError::NoMemory);
367-
checkError(2, "\xDE\x00\x01\xA1H\x01", DeserializationError::Ok);
366+
SECTION("{Hello:1}") {
367+
checkError(1, "\xDE\x00\x01\xA5Hello\x01",
368+
DeserializationError::NoMemory);
369+
checkError(2, "\xDE\x00\x01\xA5Hello\x01", DeserializationError::Ok);
368370
}
369-
SECTION("{H:1,W:2}") {
370-
checkError(2, "\xDE\x00\x02\xA1H\x01\xA1W\x02",
371+
SECTION("{Hello:1,World:2}") {
372+
checkError(2, "\xDE\x00\x02\xA5Hello\x01\xA5World\x02",
371373
DeserializationError::NoMemory);
372-
checkError(3, "\xDE\x00\x02\xA1H\x01\xA1W\x02", DeserializationError::Ok);
374+
checkError(3, "\xDE\x00\x02\xA5Hello\x01\xA5World\x02",
375+
DeserializationError::Ok);
373376
}
374377
}
375378

@@ -382,8 +385,8 @@ TEST_CASE("deserializeMsgPack() under memory constaints") {
382385
DeserializationError::NoMemory);
383386
checkError(2, "\xDF\x00\x00\x00\x01\xA1H\x01", DeserializationError::Ok);
384387
}
385-
SECTION("{H:1,W:2}") {
386-
checkError(2, "\xDF\x00\x00\x00\x02\xA1H\x01\xA1W\x02",
388+
SECTION("{Hello:1,World:2}") {
389+
checkError(2, "\xDF\x00\x00\x00\x02\xA5Hello\x01\xA5World\x02",
387390
DeserializationError::NoMemory);
388391
checkError(3, "\xDF\x00\x00\x00\x02\xA1H\x01\xA1W\x02",
389392
DeserializationError::Ok);

‎extras/tests/ResourceManager/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ add_executable(ResourceManagerTests
88
saveString.cpp
99
shrinkToFit.cpp
1010
size.cpp
11+
StringBuffer.cpp
1112
StringBuilder.cpp
1213
swap.cpp
1314
)

‎extras/tests/ResourceManager/StringBuffer.cpp‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
// ArduinoJson - https://arduinojson.org
2+
// Copyright © 2014-2025, Benoit BLANCHON
3+
// MIT License
4+
5+
#include <ArduinoJson/Memory/StringBuffer.hpp>
6+
#include <catch.hpp>
7+
8+
#include "Allocators.hpp"
9+
#include "Literals.hpp"
10+
11+
using namespace ArduinoJson::detail;
12+
13+
TEST_CASE("StringBuffer") {
14+
SpyingAllocator spy;
15+
ResourceManager resources(&spy);
16+
StringBuffer sb(&resources);
17+
VariantData variant;
18+
19+
SECTION("Tiny string") {
20+
auto ptr = sb.reserve(3);
21+
strcpy(ptr, "hi!");
22+
sb.save(&variant);
23+
24+
REQUIRE(variant.type() == VariantType::TinyString);
25+
REQUIRE(variant.asString() == "hi!");
26+
}
27+
28+
SECTION("Tiny string can't contain NUL") {
29+
auto ptr = sb.reserve(3);
30+
memcpy(ptr, "a\0b", 3);
31+
sb.save(&variant);
32+
33+
REQUIRE(variant.type() == VariantType::OwnedString);
34+
35+
auto str = variant.asString();
36+
REQUIRE(str.size() == 3);
37+
REQUIRE(str.c_str()[0] == 'a');
38+
REQUIRE(str.c_str()[1] == 0);
39+
REQUIRE(str.c_str()[2] == 'b');
40+
}
41+
42+
SECTION("Tiny string can't have 4 characters") {
43+
auto ptr = sb.reserve(4);
44+
strcpy(ptr, "alfa");
45+
sb.save(&variant);
46+
47+
REQUIRE(variant.type() == VariantType::OwnedString);
48+
REQUIRE(variant.asString() == "alfa");
49+
}
50+
}

‎extras/tests/ResourceManager/StringBuilder.cpp‎

Lines changed: 38 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@
66
#include <catch.hpp>
77

88
#include "Allocators.hpp"
9-
#include "Literals.hpp"
109

10+
using namespace ArduinoJson;
1111
using namespace ArduinoJson::detail;
1212

1313
TEST_CASE("StringBuilder") {
@@ -22,13 +22,31 @@ TEST_CASE("StringBuilder") {
2222
str.startString();
2323
str.save(&data);
2424

25-
REQUIRE(resources.size() == sizeofString(""));
2625
REQUIRE(resources.overflowed() == false);
27-
REQUIRE(spyingAllocator.log() ==
28-
AllocatorLog{
29-
Allocate(sizeofStringBuffer()),
30-
Reallocate(sizeofStringBuffer(), sizeofString("")),
31-
});
26+
REQUIRE(spyingAllocator.log() == AllocatorLog{
27+
Allocate(sizeofStringBuffer()),
28+
});
29+
REQUIRE(data.type() == VariantType::TinyString);
30+
}
31+
32+
SECTION("Tiny string") {
33+
StringBuilder str(&resources);
34+
35+
str.startString();
36+
str.append("url");
37+
38+
REQUIRE(str.isValid() == true);
39+
REQUIRE(str.str() == "url");
40+
REQUIRE(spyingAllocator.log() == AllocatorLog{
41+
Allocate(sizeofStringBuffer()),
42+
});
43+
44+
VariantData data;
45+
str.save(&data);
46+
47+
REQUIRE(resources.overflowed() == false);
48+
REQUIRE(data.type() == VariantType::TinyString);
49+
REQUIRE(data.asString() == "url");
3250
}
3351

3452
SECTION("Short string fits in first allocation") {
@@ -98,12 +116,12 @@ TEST_CASE("StringBuilder") {
98116
}
99117
}
100118

101-
static const char* saveString(StringBuilder& builder, const char* s) {
119+
static JsonString saveString(StringBuilder& builder, const char* s) {
102120
VariantData data;
103121
builder.startString();
104122
builder.append(s);
105123
builder.save(&data);
106-
return data.asString().c_str();
124+
return data.asString();
107125
}
108126

109127
TEST_CASE("StringBuilder::save() deduplicates strings") {
@@ -116,9 +134,9 @@ TEST_CASE("StringBuilder::save() deduplicates strings") {
116134
auto s2 = saveString(builder, "world");
117135
auto s3 = saveString(builder, "hello");
118136

119-
REQUIRE(s1 == "hello"_s);
120-
REQUIRE(s2 == "world"_s);
121-
REQUIRE(+s1 == +s3); // same address
137+
REQUIRE(s1 == "hello");
138+
REQUIRE(s2 == "world");
139+
REQUIRE(+s1.c_str() == +s3.c_str()); // same address
122140

123141
REQUIRE(spy.log() ==
124142
AllocatorLog{
@@ -134,9 +152,9 @@ TEST_CASE("StringBuilder::save() deduplicates strings") {
134152
auto s1 = saveString(builder, "hello world");
135153
auto s2 = saveString(builder, "hello");
136154

137-
REQUIRE(s1 == "hello world"_s);
138-
REQUIRE(s2 == "hello"_s);
139-
REQUIRE(+s2 != +s1); // different address
155+
REQUIRE(s1 == "hello world");
156+
REQUIRE(s2 == "hello");
157+
REQUIRE(+s2.c_str() != +s1.c_str()); // different address
140158

141159
REQUIRE(spy.log() ==
142160
AllocatorLog{
@@ -149,18 +167,18 @@ TEST_CASE("StringBuilder::save() deduplicates strings") {
149167

150168
SECTION("Don't overrun") {
151169
auto s1 = saveString(builder, "hello world");
152-
auto s2 = saveString(builder, "wor");
170+
auto s2 = saveString(builder, "worl");
153171

154-
REQUIRE(s1 == "hello world"_s);
155-
REQUIRE(s2 == "wor"_s);
156-
REQUIRE(s2 != s1);
172+
REQUIRE(s1 == "hello world");
173+
REQUIRE(s2 == "worl");
174+
REQUIRE(s2.c_str() != s1.c_str()); // different address
157175

158176
REQUIRE(spy.log() ==
159177
AllocatorLog{
160178
Allocate(sizeofStringBuffer()),
161179
Reallocate(sizeofStringBuffer(), sizeofString("hello world")),
162180
Allocate(sizeofStringBuffer()),
163-
Reallocate(sizeofStringBuffer(), sizeofString("wor")),
181+
Reallocate(sizeofStringBuffer(), sizeofString("worl")),
164182
});
165183
}
166184
}

0 commit comments

Comments
 (0)