Fix lexer data structure

Author: soutaro

include/rbs/lexer.h

@@ -126,20 +126,26 @@ typedef struct {
  * The lexer state is the curren token.
  *
  * ```
- * ... "a string token"
- *    ^                      start position
- *          ^                current position
- *     ~~~~~~                Token => "a str
+ #.   0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6
+ * ... " a   s t r i n g   t o k e n "
+ *    ^                                   start position (0)
+ *                ^                       current position (6)
+ *                 ^                      current character ('i', bytes = 1)
+ *     ~~~~~~~~~~~                        Token => "a str
  * ```
  * */
 typedef struct {
     rbs_string_t string;
-    int start_pos;            /* The character position that defines the start of the input */
-    int end_pos;              /* The character position that defines the end of the input */
-    rbs_position_t current;   /* The current position */
-    rbs_position_t start;     /* The start position of the current token */
+    int start_pos;          /* The character position that defines the start of the input */
+    int end_pos;            /* The character position that defines the end of the input */
+    rbs_position_t current; /* The current position: just before the current_character */
+    rbs_position_t start;   /* The start position of the current token */
+
+    unsigned int current_code_point; /* Current character code point */
+    size_t current_character_bytes;  /* Current character byte length (0 or 1~4) */
+
     bool first_token_of_line; /* This flag is used for tLINECOMMENT */
-    unsigned int last_char;   /* Last peeked character */
+
     const rbs_encoding_t *encoding;
 } rbs_lexer_t;
 
@@ -159,15 +165,23 @@ int rbs_token_bytes(rbs_token_t tok);
 const char *rbs_token_type_str(enum RBSTokenType type);
 
 /**
- * Read next character.
+ * Returns the next character.
  * */
 unsigned int rbs_peek(rbs_lexer_t *lexer);
 
 /**
- * Skip one character.
+ * Advances the current position by one character.
  * */
 void rbs_skip(rbs_lexer_t *lexer);
 
+/**
+ * Read next character and store the codepoint and byte length to the given pointers.
+ * 
+ * This doesn't update the lexer state.
+ * Returns `true` if succeeded, or `false` if reached to EOF.
+ * */
+bool rbs_next_char(rbs_lexer_t *lexer, unsigned int *codepoint, size_t *bytes);
+
 /**
  * Skip n characters.
  * */
@@ -187,4 +201,6 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer);
 
 void rbs_print_token(rbs_token_t tok);
 
+void rbs_print_lexer(rbs_lexer_t *lexer);
+
 #endif

src/lexstate.c

@@ -1,4 +1,6 @@
+#include "rbs/defines.h"
 #include "rbs/lexer.h"
+#include "rbs/util/rbs_assert.h"
 
 static const char *RBS_TOKENTYPE_NAMES[] = {
     "NullType",
@@ -112,17 +114,60 @@ int rbs_token_bytes(rbs_token_t tok) {
 }
 
 unsigned int rbs_peek(rbs_lexer_t *lexer) {
-    if (lexer->current.char_pos == lexer->end_pos) {
-        lexer->last_char = '\0';
-        return 0;
+    return lexer->current_code_point;
+}
+
+bool rbs_next_char(rbs_lexer_t *lexer, unsigned int *codepoint, size_t *byte_len) {
+    if (RBS_UNLIKELY(lexer->current.char_pos == lexer->end_pos)) {
+        return false;
+    }
+
+    const char *start = lexer->string.start + lexer->current.byte_pos;
+
+    // Fast path for ASCII (single-byte) characters
+    if ((unsigned int) *start < 128) {
+        *codepoint = (unsigned int) *start;
+        *byte_len = 1;
+        return true;
+    }
+
+    *byte_len = lexer->encoding->char_width((const uint8_t *) start, (ptrdiff_t) (lexer->string.end - start));
+
+    if (*byte_len == 1) {
+        *codepoint = (unsigned int) *start;
     } else {
-        rbs_string_t str = rbs_string_new(
-            lexer->string.start + lexer->current.byte_pos,
-            lexer->string.end
-        );
-        unsigned int c = rbs_utf8_string_to_codepoint(str);
-        lexer->last_char = c;
-        return c;
+        *codepoint = 12523; // Dummy data for "ル" from "ルビー" (Ruby) in Unicode
+    }
+
+    return true;
+}
+
+void rbs_skip(rbs_lexer_t *lexer) {
+    rbs_assert(lexer->current_character_bytes > 0, "rbs_skip called with current_character_bytes == 0");
+
+    if (RBS_UNLIKELY(lexer->current_code_point == '\0')) {
+        return;
+    }
+
+    unsigned int codepoint;
+    size_t byte_len;
+
+    lexer->current.byte_pos += lexer->current_character_bytes;
+    lexer->current.char_pos += 1;
+    if (lexer->current_code_point == '\n') {
+        lexer->current.line += 1;
+        lexer->current.column = 0;
+        lexer->first_token_of_line = true;
+    } else {
+        lexer->current.column += 1;
+    }
+
+    if (rbs_next_char(lexer, &codepoint, &byte_len)) {
+        lexer->current_code_point = codepoint;
+        lexer->current_character_bytes = byte_len;
+    } else {
+        lexer->current_character_bytes = 1;
+        lexer->current_code_point = '\0';
     }
 }
 
@@ -156,35 +201,8 @@ rbs_token_t rbs_next_eof_token(rbs_lexer_t *lexer) {
     }
 }
 
-void rbs_skip(rbs_lexer_t *lexer) {
-    if (!lexer->last_char) {
-        rbs_peek(lexer);
-    }
-
-    size_t byte_len;
-
-    if (lexer->last_char == '\0') {
-        byte_len = 1;
-    } else {
-        const char *start = lexer->string.start + lexer->current.byte_pos;
-        byte_len = lexer->encoding->char_width((const uint8_t *) start, (ptrdiff_t) (lexer->string.end - start));
-    }
-
-    lexer->current.char_pos += 1;
-    lexer->current.byte_pos += byte_len;
-
-    if (lexer->last_char == '\n') {
-        lexer->current.line += 1;
-        lexer->current.column = 0;
-        lexer->first_token_of_line = true;
-    } else {
-        lexer->current.column += 1;
-    }
-}
-
 void rbs_skipn(rbs_lexer_t *lexer, size_t size) {
     for (size_t i = 0; i < size; i++) {
-        rbs_peek(lexer);
         rbs_skip(lexer);
     }
 }

src/parser.c

@@ -7,6 +7,7 @@
 #include <string.h>
 
 #include "rbs/defines.h"
+#include "rbs/lexer.h"
 #include "rbs/string.h"
 #include "rbs/util/rbs_unescape.h"
 #include "rbs/util/rbs_buffer.h"
@@ -3456,6 +3457,14 @@ void rbs_print_token(rbs_token_t tok) {
     );
 }
 
+void rbs_print_lexer(rbs_lexer_t *lexer) {
+    printf("Lexer: (range = %d...%d, encoding = %s\n", lexer->start_pos, lexer->end_pos, lexer->encoding->name);
+    printf("  start = { char_pos = %d, byte_pos = %d }\n", lexer->start.char_pos, lexer->start.byte_pos);
+    printf("  current = { char_pos = %d, byte_pos = %d }\n", lexer->current.char_pos, lexer->current.byte_pos);
+    printf("  character = { code_point = %d (%c), bytes = %zu }\n", lexer->current_code_point, lexer->current_code_point < 256 ? lexer->current_code_point : '?', lexer->current_character_bytes);
+    printf("  first_token_of_line = %s\n", lexer->first_token_of_line ? "true" : "false");
+}
+
 rbs_ast_comment_t *rbs_parser_get_comment(rbs_parser_t *parser, int subject_line) {
     int comment_line = subject_line - 1;
 
@@ -3484,14 +3493,28 @@ rbs_lexer_t *rbs_lexer_new(rbs_allocator_t *allocator, rbs_string_t string, cons
         .end_pos = end_pos,
         .current = start_position,
         .start = { 0 },
-        .first_token_of_line = false,
-        .last_char = 0,
+        .first_token_of_line = true,
+        .current_character_bytes = 0,
+        .current_code_point = '\0',
         .encoding = encoding,
     };
 
-    rbs_skipn(lexer, start_pos);
+    unsigned int codepoint;
+    size_t bytes;
+
+    if (rbs_next_char(lexer, &codepoint, &bytes)) {
+        lexer->current_code_point = codepoint;
+        lexer->current_character_bytes = bytes;
+    } else {
+        lexer->current_code_point = '\0';
+        lexer->current_character_bytes = 1;
+    }
+
+    if (start_pos > 0) {
+        rbs_skipn(lexer, start_pos);
+    }
+
     lexer->start = lexer->current;
-    lexer->first_token_of_line = lexer->current.column == 0;
 
     return lexer;
 }

src/string.c

@@ -1,4 +1,5 @@
 #include "rbs/string.h"
+#include "rbs/defines.h"
 
 #include <stdlib.h>
 #include <string.h>
@@ -14,7 +15,7 @@ unsigned int rbs_utf8_string_to_codepoint(const rbs_string_t string) {
 
     if (s >= end) return 0; // End of string
 
-    if ((*s & 0x80) == 0) {
+    if (RBS_LIKELY((*s & 0x80) == 0)) {
         // Single byte character (0xxxxxxx)
         return *s;
     } else if ((*s & 0xE0) == 0xC0) {