Refactor frontend and error diagnostic

This patch completely separate the preprocessor functionality
from scanner-less parser to indepadent units, which allows compiler to
expand and parse nested function-like macro, multi-token object-like
macro, and more. Furthermore, the error diagnostic is rewritten to
better allow user to find out where and what lexeme causes compiler to
panic.

Author: ChAoSUnItY

.github/workflows/main.yml

@@ -56,6 +56,45 @@ jobs:
           make check-sanitizer || exit 1
           make check || exit 1
 
+  preprocessor-host:
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        compiler: [gcc, clang]
+        architecture: [arm, riscv]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Download dependencies
+        run: |
+          sudo apt-get update -q -y
+          sudo apt-get install -q -y graphviz jq
+          sudo apt-get install -q -y qemu-user
+          sudo apt-get install -q -y build-essential
+      - name: Configurate config
+        run: |
+          make distclean config ARCH=${{ matrix.architecture }}
+      - name: Preprocess stage 1 source code
+        env:
+          CC: ${{ matrix.compiler }}
+        run: |
+          make out/shecc
+          ./out/shecc -E src/main.c > ./out/out.c
+      - name: Build stage 1 artifact
+        run: |
+          ./out/shecc --no-libc -o out/shecc-stage1.elf ./out/out.c
+          chmod a+x ./out/shecc-stage1.elf
+      - name: Preprocess stage 2 source code
+        run: |
+          ./out/shecc-stage1.elf -E src/main.c > ./out/out.c
+      - name: Build stage 2 artifact
+        run: |
+          ./out/shecc-stage1.elf --no-libc -o out/shecc-stage2.elf ./out/out.c
+          chmod a+x ./out/shecc-stage2.elf
+      - name: Test stage 2 artifact
+        run: |
+          make check-stage2 || exit 1
+
   coding-style:
     runs-on: ubuntu-24.04
     steps:

lib/c.c

@@ -16,13 +16,18 @@
 #define INT_MAX 0x7fffffff
 #define INT_MIN 0x80000000
 
+#define SEEK_SET 0
+#define SEEK_CUR 1
+#define SEEK_END 2
+
 #if defined(__arm__)
 #define __SIZEOF_POINTER__ 4
 #define __syscall_exit 1
 #define __syscall_read 3
 #define __syscall_write 4
 #define __syscall_close 6
 #define __syscall_open 5
+#define __syscall_lseek 19
 #define __syscall_mmap2 192
 #define __syscall_munmap 91
 
@@ -34,6 +39,7 @@
 #define __syscall_close 57
 #define __syscall_open 1024
 #define __syscall_openat 56
+#define __syscall_lseek 62
 #define __syscall_mmap2 222
 #define __syscall_munmap 215
 
@@ -584,6 +590,30 @@ int fputc(int c, FILE *stream)
     return c;
 }
 
+int fseek(FILE *stream, int offset, int whence)
+{
+#if defined(__arm__)
+    __syscall(__syscall_lseek, stream, offset, whence);
+#elif defined(__riscv)
+    /* No need to offset */
+    __syscall(__syscall_lseek, stream, 0, offset, NULL, whence);
+#endif
+    return 0;
+}
+
+int ftell(FILE *stream)
+{
+#if defined(__arm__)
+    return __syscall(__syscall_lseek, stream, 0, SEEK_CUR);
+#elif defined(__riscv)
+    int result;
+    __syscall(__syscall_lseek, stream, 0, 0, &result, SEEK_CUR);
+    return result;
+#else
+#error "Unsupported ftell support for current platform"
+#endif
+}
+
 /* Non-portable: Assume page size is 4KiB */
 #define PAGESIZE 4096
 

src/defs.h

@@ -10,6 +10,18 @@
 
 /* definitions */
 
+#define DEBUG_BUILD false
+
+/* Common macro functions */
+#define is_whitespace(c) (c == ' ' || c == '\t')
+#define is_newline(c) (c == '\r' || c == '\n')
+#define is_alnum(c)                                      \
+    ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || \
+     (c >= '0' && c <= '9') || (c == '_'))
+#define is_digit(c) ((c >= '0' && c <= '9'))
+#define is_hex(c) \
+    (is_digit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'))
+
 /* Limitations */
 #define MAX_TOKEN_LEN 256
 #define MAX_ID_LEN 64
@@ -26,14 +38,12 @@
 #define MAX_BB_DOM_SUCC 64
 #define MAX_BB_RDOM_SUCC 256
 #define MAX_GLOBAL_IR 256
-#define MAX_SOURCE 1048576
 #define MAX_CODE 262144
 #define MAX_DATA 262144
 #define MAX_SYMTAB 65536
 #define MAX_STRTAB 65536
 #define MAX_HEADER 1024
 #define MAX_SECTION 1024
-#define MAX_ALIASES 128
 #define MAX_CONSTANTS 1024
 #define MAX_CASES 128
 #define MAX_NESTING 128
@@ -46,7 +56,7 @@
 #define SMALL_ARENA_SIZE 65536    /* 64 KiB - for small allocations */
 #define LARGE_ARENA_SIZE 524288   /* 512 KiB - for instruction arena */
 #define DEFAULT_FUNCS_SIZE 64
-#define DEFAULT_INCLUSIONS_SIZE 16
+#define DEFAULT_SRC_FILE_COUNT 8
 
 /* Arena compaction bitmask flags for selective memory reclamation */
 #define COMPACT_ARENA_BLOCK 0x01   /* BLOCK_ARENA - variables/blocks */
@@ -113,6 +123,7 @@ typedef struct {
 /* lexer tokens */
 typedef enum {
     T_start, /* FIXME: Unused, intended for lexer state machine init */
+    T_eof,   /* end-of-file (EOF) */
     T_numeric,
     T_identifier,
     T_comma,  /* , */
@@ -161,7 +172,6 @@ typedef enum {
     T_question,      /* ? */
     T_colon,         /* : */
     T_semicolon,     /* ; */
-    T_eof,           /* end-of-file (EOF) */
     T_ampersand,     /* & */
     T_return,
     T_if,
@@ -193,38 +203,36 @@ typedef enum {
     T_cppd_endif,
     T_cppd_ifdef,
     T_cppd_ifndef,
-    T_cppd_pragma
-} token_t;
+    T_cppd_pragma,
+    /* C pre-processor specific, these kinds
+     * will be removed after pre-processing is done.
+     */
+    T_newline,
+    T_backslash,
+    T_whitespace,
+    T_tab
+} token_kind_t;
 
 /* Source location tracking for better error reporting */
 typedef struct {
+    int pos; /* raw source file position */
+    int len; /* length of token */
     int line;
     int column;
     char *filename;
 } source_location_t;
 
-/* Token structure with metadata for enhanced lexing */
-typedef struct token_info {
-    token_t type;
-    char value[MAX_TOKEN_LEN];
+typedef struct token {
+    token_kind_t kind;
+    char *literal;
     source_location_t location;
-    struct token_info *next; /* For freelist management */
-} token_info_t;
-
-/* Token freelist for memory reuse */
-typedef struct {
-    token_info_t *freelist;
-    int allocated_count;
-} token_pool_t;
+    struct token *next;
+} token_t;
 
-/* Token buffer for improved lookahead */
-#define TOKEN_BUFFER_SIZE 8
-typedef struct {
-    token_info_t *tokens[TOKEN_BUFFER_SIZE];
-    int head;
-    int tail;
-    int count;
-} token_buffer_t;
+typedef struct token_stream {
+    token_t *head;
+    token_t *tail;
+} token_stream_t;
 
 /* String pool for identifier deduplication */
 typedef struct {
@@ -369,7 +377,7 @@ struct var {
     int in_loop;
     struct var *base;
     int subscript;
-    struct var *subscripts[64];
+    struct var *subscripts[128];
     int subscripts_idx;
     rename_t rename;
     ref_block_list_t ref_block_list; /* blocks which kill variable */
@@ -388,25 +396,13 @@ struct var {
     int use_count;  /* Number of times variable is used */
 };
 
-typedef struct {
-    char name[MAX_VAR_LEN];
-    bool is_variadic;
-    int start_source_idx;
-    var_t param_defs[MAX_PARAMS];
-    int num_param_defs;
-    int params[MAX_PARAMS];
-    int num_params;
-    bool disabled;
-} macro_t;
-
 typedef struct func func_t;
 
 /* block definition */
 struct block {
     var_list_t locals;
     struct block *parent;
     func_t *func;
-    macro_t *macro;
     struct block *next;
 };
 
@@ -460,13 +456,6 @@ typedef struct {
     type_t *type;
 } lvalue_t;
 
-/* alias for #defines */
-typedef struct {
-    char alias[MAX_VAR_LEN];
-    char value[MAX_VAR_LEN];
-    bool disabled;
-} alias_t;
-
 /* constants for enums */
 typedef struct {
     char alias[MAX_VAR_LEN];