Skip to content

Optimize parsing packed repeated bool fields. #25082

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
copybara-service merged 1 commit into from
Jan 7, 2026
+173 −105
Merged

Optimize parsing packed repeated bool fields. #25082

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 22 additions & 16 deletions src/google/protobuf/lite_unittest.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,11 @@

// Author: [email protected] (Kenton Varda)

#include <climits>
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <limits>
#include <memory>
Expand All @@ -24,8 +27,6 @@
#include "absl/strings/string_view.h"
#include "google/protobuf/generated_enum_util.h"
#include "google/protobuf/io/coded_stream.h"
#include "google/protobuf/io/zero_copy_stream.h"
#include "google/protobuf/io/zero_copy_stream_impl.h"
#include "google/protobuf/io/zero_copy_stream_impl_lite.h"
#include "google/protobuf/map_lite_unittest.pb.h"
#include "google/protobuf/map_test_util.h"
Expand Down Expand Up @@ -347,19 +348,24 @@ TYPED_TEST(LiteTest, AllLite7) {
TYPED_TEST(LiteTest, AllLite8) {
TypeParam data;

{
proto2_unittest::TestPackedTypesLite message, message2;
proto2_unittest::TestEmptyMessageLite empty_message;
TestUtilLite::ExpectPackedClear(message);
TestUtilLite::SetPackedFields(&message);
data = SerializeAs<TypeParam>(message);
ASSERT_TRUE(ParseFrom(data, empty_message));
data = SerializeAs<TypeParam>(empty_message);
ASSERT_TRUE(ParseFrom(data, message2));
data = message2.SerializeAsString();
TestUtilLite::ExpectPackedFieldsSet(message2);
message.Clear();
TestUtilLite::ExpectPackedClear(message);
for (bool large_values : {true, false}) {
SCOPED_TRACE(large_values);
for (int repetitions : {1, 100}) {
SCOPED_TRACE(repetitions);

proto2_unittest::TestPackedTypesLite message, message2;
proto2_unittest::TestEmptyMessageLite empty_message;
TestUtilLite::ExpectPackedClear(message);
TestUtilLite::SetPackedFields(&message, large_values, repetitions);
data = SerializeAs<TypeParam>(message);
ASSERT_TRUE(ParseFrom(data, empty_message));
data = SerializeAs<TypeParam>(empty_message);
ASSERT_TRUE(ParseFrom(data, message2));
data = message2.SerializeAsString();
TestUtilLite::ExpectPackedFieldsSet(message2, large_values, repetitions);
message.Clear();
TestUtilLite::ExpectPackedClear(message);
}
}
}

Expand Down
20 changes: 20 additions & 0 deletions src/google/protobuf/parse_context.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -789,6 +789,26 @@ int CountVarintsAssumingLargeArray(const char* ptr, const char* end) {
(0x8080808080808080 << ((ptr - limit) * 8)));
}

bool VerifyBoolsAssumingLargeArray(const char* ptr, const char* end) {
ABSL_DCHECK_GE(end - ptr, int{sizeof(uint64_t)});

// Verify whole blocks, except for the last one.
uint64_t bit_or = 0;
const char* const limit = end - sizeof(uint64_t);
while (ptr < limit) {
uint64_t block;
std::memcpy(&block, ptr, 8);
bit_or |= block;
ptr += 8;
}
// Verify the last, possibly incomplete block.
uint64_t block;
std::memcpy(&block, limit, 8);
bit_or |= block;

return (bit_or & ~0x0101010101010101) == 0;
}

} // namespace internal
} // namespace protobuf
} // namespace google
Expand Down
22 changes: 21 additions & 1 deletion src/google/protobuf/parse_context.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include <type_traits>
#include <utility>

#include "absl/base/casts.h"
#include "absl/base/config.h"
#include "absl/base/prefetch.h"
#include "absl/log/absl_check.h"
Expand Down Expand Up @@ -70,6 +71,10 @@ inline void WriteLengthDelimited(uint32_t num, absl::string_view val,
// of valid varints.
int CountVarintsAssumingLargeArray(const char* ptr, const char* end);

// Checks if each byte in the array is a valid representation for a bool, i.e.
// 0 or 1, assuming that end - ptr >= 8. Optimized for the result being true.
bool VerifyBoolsAssumingLargeArray(const char* ptr, const char* end);


// The basic abstraction the parser is designed for is a slight modification
// of the ZeroCopyInputStream (ZCIS) abstraction. A ZCIS presents a serialized
Expand Down Expand Up @@ -1437,6 +1442,21 @@ const char* EpsCopyInputStream::ReadPackedVarintArrayWithField(
// Assume that varint are valid and just count the number of bytes with
// continuation bit not set. In a valid varint there is only 1 such byte.
if (end - ptr >= 16) {
if constexpr (std::is_same_v<T, bool> && sizeof(bool) == sizeof(uint8_t)) {
if (absl::bit_cast<uint8_t>(false) == 0 && // Not constexpr on MSVC.
absl::bit_cast<uint8_t>(true) == 1) {
if (VerifyBoolsAssumingLargeArray(ptr, end)) {
// Each byte is 0 or 1.
const int count = end - ptr;
out.ReserveWithArena(arena, out.size() + count);
T* x = out.AddNAlreadyReserved(count);
// For T being bool, conv must be equivalent to a conversion to bool
// (zigzag encoding is not applicable), so it can be skipped.
std::memcpy(x, ptr, count);
return end;
}
}
}
int count = CountVarintsAssumingLargeArray(ptr, end);
if (count == end - ptr) {
// We have exactly one element per byte, so avoid the costly varint
Expand All @@ -1460,7 +1480,7 @@ const char* EpsCopyInputStream::ReadPackedVarintArrayWithField(
});
int new_size = x - out.data();
ABSL_DCHECK_LE(new_size, old_size + count);
// We may have overreserved if the the data are truncated or malformed,
// We may have overreserved if the data are truncated or malformed,
// so set the actual size to avoid exposing uninitialized memory.
out.Truncate(new_size);
}
Expand Down
179 changes: 100 additions & 79 deletions src/google/protobuf/test_util_lite.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -613,49 +613,65 @@ void TestUtilLite::ExpectRepeatedFieldsModified(
// -------------------------------------------------------------------

void TestUtilLite::SetPackedVarintFields(unittest::TestPackedTypesLite* message,
bool large_values) {
const uint64_t kMask = large_values ? 0xFFFFFFFFFFFFFFFF : 0x7F;
message->add_packed_int32(601 & kMask);
message->add_packed_int64(602 & kMask);
message->add_packed_uint32(603 & kMask);
message->add_packed_uint64(604 & kMask);
message->add_packed_sint32(605 & kMask);
message->add_packed_sint64(606 & kMask);
message->add_packed_enum(unittest::FOREIGN_LITE_BAR);
// add a second one of each field
message->add_packed_int32(701 & kMask);
message->add_packed_int64(702 & kMask);
message->add_packed_uint32(703 & kMask);
message->add_packed_uint64(704 & kMask);
message->add_packed_sint32(705 & kMask);
message->add_packed_sint64(706 & kMask);
message->add_packed_enum(unittest::FOREIGN_LITE_BAZ);
bool large_values, int repetitions) {
for (int i = 0; i < repetitions; ++i) {
const uint64_t kMask = large_values ? 0xFFFFFFFFFFFFFFFF : 0x7F;
message->add_packed_int32(601 & kMask);
message->add_packed_int64(602 & kMask);
message->add_packed_uint32(603 & kMask);
message->add_packed_uint64(604 & kMask);
message->add_packed_sint32(605 & kMask);
message->add_packed_sint64(606 & kMask);
message->add_packed_enum(unittest::FOREIGN_LITE_BAR);
// add a second one of each field
message->add_packed_int32(701 & kMask);
message->add_packed_int64(702 & kMask);
message->add_packed_uint32(703 & kMask);
message->add_packed_uint64(704 & kMask);
message->add_packed_sint32(705 & kMask);
message->add_packed_sint64(706 & kMask);
message->add_packed_enum(unittest::FOREIGN_LITE_BAZ);
}
}

// -------------------------------------------------------------------

void TestUtilLite::SetPackedBoolField(unittest::TestPackedTypesLite* message) {
message->add_packed_bool(true);
message->add_packed_bool(false);
void TestUtilLite::SetPackedBoolField(unittest::TestPackedTypesLite* message,
int repetitions) {
for (int i = 0; i < repetitions; ++i) {
message->add_packed_bool(true);
message->add_packed_bool(false);
}
}

// -------------------------------------------------------------------

void TestUtilLite::SetPackedFixedFields(
unittest::TestPackedTypesLite* message) {
message->add_packed_fixed32(607);
message->add_packed_fixed64(608);
message->add_packed_sfixed32(609);
message->add_packed_sfixed64(610);
message->add_packed_float(611);
message->add_packed_double(612);
// add a second one of each field
message->add_packed_fixed32(707);
message->add_packed_fixed64(708);
message->add_packed_sfixed32(709);
message->add_packed_sfixed64(710);
message->add_packed_float(711);
message->add_packed_double(712);
void TestUtilLite::SetPackedFixedFields(unittest::TestPackedTypesLite* message,
int repetitions) {
for (int i = 0; i < repetitions; ++i) {
message->add_packed_fixed32(607);
message->add_packed_fixed64(608);
message->add_packed_sfixed32(609);
message->add_packed_sfixed64(610);
message->add_packed_float(611);
message->add_packed_double(612);
// add a second one of each field
message->add_packed_fixed32(707);
message->add_packed_fixed64(708);
message->add_packed_sfixed32(709);
message->add_packed_sfixed64(710);
message->add_packed_float(711);
message->add_packed_double(712);
}
}

// -------------------------------------------------------------------

void TestUtilLite::SetPackedFields(unittest::TestPackedTypesLite* message,
bool large_values, int repetitions) {
SetPackedVarintFields(message, large_values, repetitions);
SetPackedBoolField(message, repetitions);
SetPackedFixedFields(message, repetitions);
}

// -------------------------------------------------------------------
Expand All @@ -680,51 +696,56 @@ void TestUtilLite::ModifyPackedFields(unittest::TestPackedTypesLite* message) {
// -------------------------------------------------------------------

void TestUtilLite::ExpectPackedFieldsSet(
const unittest::TestPackedTypesLite& message) {
ASSERT_EQ(2, message.packed_int32_size());
ASSERT_EQ(2, message.packed_int64_size());
ASSERT_EQ(2, message.packed_uint32_size());
ASSERT_EQ(2, message.packed_uint64_size());
ASSERT_EQ(2, message.packed_sint32_size());
ASSERT_EQ(2, message.packed_sint64_size());
ASSERT_EQ(2, message.packed_fixed32_size());
ASSERT_EQ(2, message.packed_fixed64_size());
ASSERT_EQ(2, message.packed_sfixed32_size());
ASSERT_EQ(2, message.packed_sfixed64_size());
ASSERT_EQ(2, message.packed_float_size());
ASSERT_EQ(2, message.packed_double_size());
ASSERT_EQ(2, message.packed_bool_size());
ASSERT_EQ(2, message.packed_enum_size());

EXPECT_EQ(601, message.packed_int32(0));
EXPECT_EQ(602, message.packed_int64(0));
EXPECT_EQ(603, message.packed_uint32(0));
EXPECT_EQ(604, message.packed_uint64(0));
EXPECT_EQ(605, message.packed_sint32(0));
EXPECT_EQ(606, message.packed_sint64(0));
EXPECT_EQ(607, message.packed_fixed32(0));
EXPECT_EQ(608, message.packed_fixed64(0));
EXPECT_EQ(609, message.packed_sfixed32(0));
EXPECT_EQ(610, message.packed_sfixed64(0));
EXPECT_EQ(611, message.packed_float(0));
EXPECT_EQ(612, message.packed_double(0));
EXPECT_EQ(true, message.packed_bool(0));
EXPECT_EQ(unittest::FOREIGN_LITE_BAR, message.packed_enum(0));
const unittest::TestPackedTypesLite& message, bool large_values,
int repetitions) {
const uint64_t kMask = large_values ? 0xFFFFFFFFFFFFFFFF : 0x7F;

EXPECT_EQ(701, message.packed_int32(1));
EXPECT_EQ(702, message.packed_int64(1));
EXPECT_EQ(703, message.packed_uint32(1));
EXPECT_EQ(704, message.packed_uint64(1));
EXPECT_EQ(705, message.packed_sint32(1));
EXPECT_EQ(706, message.packed_sint64(1));
EXPECT_EQ(707, message.packed_fixed32(1));
EXPECT_EQ(708, message.packed_fixed64(1));
EXPECT_EQ(709, message.packed_sfixed32(1));
EXPECT_EQ(710, message.packed_sfixed64(1));
EXPECT_EQ(711, message.packed_float(1));
EXPECT_EQ(712, message.packed_double(1));
EXPECT_FALSE(message.packed_bool(1));
EXPECT_EQ(unittest::FOREIGN_LITE_BAZ, message.packed_enum(1));
ASSERT_EQ(repetitions * 2, message.packed_int32_size());
ASSERT_EQ(repetitions * 2, message.packed_int64_size());
ASSERT_EQ(repetitions * 2, message.packed_uint32_size());
ASSERT_EQ(repetitions * 2, message.packed_uint64_size());
ASSERT_EQ(repetitions * 2, message.packed_sint32_size());
ASSERT_EQ(repetitions * 2, message.packed_sint64_size());
ASSERT_EQ(repetitions * 2, message.packed_fixed32_size());
ASSERT_EQ(repetitions * 2, message.packed_fixed64_size());
ASSERT_EQ(repetitions * 2, message.packed_sfixed32_size());
ASSERT_EQ(repetitions * 2, message.packed_sfixed64_size());
ASSERT_EQ(repetitions * 2, message.packed_float_size());
ASSERT_EQ(repetitions * 2, message.packed_double_size());
ASSERT_EQ(repetitions * 2, message.packed_bool_size());
ASSERT_EQ(repetitions * 2, message.packed_enum_size());

for (int i = 0; i < repetitions; ++i) {
EXPECT_EQ(601 & kMask, message.packed_int32(i * 2));
EXPECT_EQ(602 & kMask, message.packed_int64(i * 2));
EXPECT_EQ(603 & kMask, message.packed_uint32(i * 2));
EXPECT_EQ(604 & kMask, message.packed_uint64(i * 2));
EXPECT_EQ(605 & kMask, message.packed_sint32(i * 2));
EXPECT_EQ(606 & kMask, message.packed_sint64(i * 2));
EXPECT_EQ(607, message.packed_fixed32(i * 2));
EXPECT_EQ(608, message.packed_fixed64(i * 2));
EXPECT_EQ(609, message.packed_sfixed32(i * 2));
EXPECT_EQ(610, message.packed_sfixed64(i * 2));
EXPECT_EQ(611, message.packed_float(i * 2));
EXPECT_EQ(612, message.packed_double(i * 2));
EXPECT_EQ(true, message.packed_bool(i * 2));
EXPECT_EQ(unittest::FOREIGN_LITE_BAR, message.packed_enum(i * 2));

EXPECT_EQ(701 & kMask, message.packed_int32(i * 2 + 1));
EXPECT_EQ(702 & kMask, message.packed_int64(i * 2 + 1));
EXPECT_EQ(703 & kMask, message.packed_uint32(i * 2 + 1));
EXPECT_EQ(704 & kMask, message.packed_uint64(i * 2 + 1));
EXPECT_EQ(705 & kMask, message.packed_sint32(i * 2 + 1));
EXPECT_EQ(706 & kMask, message.packed_sint64(i * 2 + 1));
EXPECT_EQ(707, message.packed_fixed32(i * 2 + 1));
EXPECT_EQ(708, message.packed_fixed64(i * 2 + 1));
EXPECT_EQ(709, message.packed_sfixed32(i * 2 + 1));
EXPECT_EQ(710, message.packed_sfixed64(i * 2 + 1));
EXPECT_EQ(711, message.packed_float(i * 2 + 1));
EXPECT_EQ(712, message.packed_double(i * 2 + 1));
EXPECT_FALSE(message.packed_bool(i * 2 + 1));
EXPECT_EQ(unittest::FOREIGN_LITE_BAZ, message.packed_enum(i * 2 + 1));
}
}

// -------------------------------------------------------------------
Expand Down
19 changes: 10 additions & 9 deletions src/google/protobuf/test_util_lite.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,14 @@ class TestUtilLite {
static void SetAllFields(unittest::TestAllTypesLite* message);
static void SetAllExtensions(unittest::TestAllExtensionsLite* message);
static void SetPackedVarintFields(unittest::TestPackedTypesLite* message,
bool large_values = true);
static void SetPackedBoolField(unittest::TestPackedTypesLite* message);
static void SetPackedFixedFields(unittest::TestPackedTypesLite* message);
static void SetPackedFields(unittest::TestPackedTypesLite* message) {
SetPackedVarintFields(message);
SetPackedBoolField(message);
SetPackedFixedFields(message);
}
bool large_values = true,
int repetitions = 1);
static void SetPackedBoolField(unittest::TestPackedTypesLite* message,
int repetitions = 1);
static void SetPackedFixedFields(unittest::TestPackedTypesLite* message,
int repetitions = 1);
static void SetPackedFields(unittest::TestPackedTypesLite* message,
bool large_values = true, int repetitions = 1);
static void SetPackedExtensions(unittest::TestPackedExtensionsLite* message);

// Use the repeated versions of the set_*() accessors to modify all the
Expand All @@ -55,7 +55,8 @@ class TestUtilLite {
static void ExpectAllExtensionsSet(
const unittest::TestAllExtensionsLite& message);
static void ExpectPackedFieldsSet(
const unittest::TestPackedTypesLite& message);
const unittest::TestPackedTypesLite& message, bool large_values = true,
int repetitions = 1);
static void ExpectPackedExtensionsSet(
const unittest::TestPackedExtensionsLite& message);

Expand Down
Loading