Skip to content

parser.c: Use SWAR to skip consecutive spaces #886

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
byroot merged 1 commit into from
Nov 1, 2025
+33 −16
Merged

parser.c: Use SWAR to skip consecutive spaces #886

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 32 additions & 14 deletions ext/json/ext/parser/parser.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -557,14 +557,6 @@ static uint32_t unescape_unicode(JSON_ParserState *state, const unsigned char *p

static const rb_data_type_t JSON_ParserConfig_type;

static const bool whitespace[256] = {
[' '] = 1,
['\t'] = 1,
['\n'] = 1,
['\r'] = 1,
['/'] = 1,
};

static void
json_eat_comments(JSON_ParserState *state)
{
Expand Down Expand Up @@ -607,12 +599,38 @@ json_eat_comments(JSON_ParserState *state)
static inline void
json_eat_whitespace(JSON_ParserState *state)
{
unsigned char cursor;
while (RB_UNLIKELY(whitespace[cursor = (unsigned char)peek(state)])) {
if (RB_UNLIKELY(cursor == '/')) {
json_eat_comments(state);
} else {
state->cursor++;
while (true) {
switch (peek(state)) {
case ' ':
state->cursor++;
break;
case '\n':
state->cursor++;

// Heuristic: if we see a newline, there is likely consecutive spaces after it.
#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
while (rest(state) > 8) {
uint64_t chunk;
memcpy(&chunk, state->cursor, sizeof(uint64_t));
size_t consecutive_spaces = trailing_zeros64(chunk ^ 0x2020202020202020) / CHAR_BIT;

state->cursor += consecutive_spaces;
if (consecutive_spaces != 8) {
break;
}
}
#endif
break;
case '\t':
case '\r':
state->cursor++;
break;
case '/':
json_eat_comments(state);
break;

default:
return;
}
}
}
Expand Down
3 changes: 1 addition & 2 deletions ext/json/ext/simd/simd.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@ typedef enum {
SIMD_SSE2
} SIMD_Implementation;

#ifdef JSON_ENABLE_SIMD

#ifdef __clang__
# if __has_builtin(__builtin_ctzll)
# define HAVE_BUILTIN_CTZLL 1
Expand Down Expand Up @@ -54,6 +52,7 @@ static inline int trailing_zeros(int input)
#define FORCE_INLINE
#endif

#ifdef JSON_ENABLE_SIMD

#define SIMD_MINIMUM_THRESHOLD 6

Expand Down
Loading