Skip to content

Commit 52c5234

Browse files
committed
Add GBK/CP936 conversion failure check
1 parent 5a281b1 commit 52c5234

File tree

2 files changed

+12
-3
lines changed

2 files changed

+12
-3
lines changed

src/openvic-dataloader/detail/ConvertGbk.hpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,10 @@ namespace ovdl::convert::gbk {
218218
#else
219219
#error "GBK conversion not supported on this platform"
220220
#endif
221+
if (in_size > 0 && out_size > 0 && in_buffer != end) {
222+
return lexy::buffer<Encoding, MemoryResource> { resource };
223+
}
224+
221225
return lexy::buffer<Encoding, MemoryResource> { out_builder.data(), static_cast<size_t>(out_buffer - out_builder.data()), resource };
222226
} else {
223227
return lexy::make_buffer_from_raw<Encoding, Endian>(_memory, size, resource);

src/openvic-dataloader/detail/ParseHandler.hpp

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -111,17 +111,22 @@ namespace ovdl::detail {
111111
};
112112

113113
template<detail::IsStateType State>
114-
static constexpr auto generate_gbk_state(State* state, const char* path, auto&& buffer, Encoding encoding) {
114+
static auto generate_gbk_state(State* state, const char* path, auto&& buffer, Encoding encoding) {
115115
size_t size = buffer.size();
116+
lexy::buffer<lexy::utf8_char_encoding, void> utf8_buffer =
117+
convert::gbk::make_buffer_from_raw<lexy::utf8_char_encoding>(encoding, std::move(buffer).release(), size);
118+
if (utf8_buffer.data() == nullptr) {
119+
state->logger().error("conversion from GBK to UTF-8 failed");
120+
}
116121
if (path[0] != '\0') {
117122
*state = {
118123
path,
119-
convert::gbk::make_buffer_from_raw<lexy::utf8_char_encoding>(encoding, std::move(buffer).release(), size),
124+
std::move(utf8_buffer),
120125
encoding
121126
};
122127
return;
123128
}
124-
*state = { convert::gbk::make_buffer_from_raw<lexy::utf8_char_encoding>(encoding, std::move(buffer).release(), size), encoding };
129+
*state = { std::move(utf8_buffer), encoding };
125130
};
126131

127132
template<detail::IsStateType State>

0 commit comments

Comments
 (0)