Fix build

Author: jbampton

CMakeLists.txt

@@ -61,19 +61,15 @@ set(BUILTINS_HEADERS
 
 # Set lexer source and headers
 set(LEXER_SOURCE
-        src/lexer/tokenizer/token.c
+        src/parser/lexer/token.c
 
-        src/lexer/tokenizer/tokenizer.c
-
-        src/lexer/lexer.c
+        src/parser/lexer/lexer.c
 )
 
 set(LEXER_HEADERS
-        src/lexer/tokenizer/token.h
-
-        src/lexer/tokenizer/tokenizer.h
+        src/parser/lexer/token.h
 
-        src/lexer/lexer.h
+        src/parser/lexer/lexer.h
 )
 
 # Set AST source and headers
@@ -100,7 +96,7 @@ set(PARSER_HEADERS
 
 # Set Main executables for lexer, ast, and parser
 set(LEXER_MAIN
-        src/lexer/main.c
+        src/parser/lexer/main.c
 )
 
 set(AST_MAIN

Dockerfile

@@ -2,7 +2,7 @@ FROM alpine:latest
 LABEL name="The One Programming Language"
 
 # LLVM version
-ARG LLVM_VERSION=12.0.1
+ARG LLVM_VERSION=16.0.3
 
 # LLVM dependencies
 RUN apk --no-cache add \
@@ -58,8 +58,7 @@ RUN cmake --build ./build --config Debug --target all -j 6 --
 WORKDIR /One/build
 
 # Running example input.one
-RUN ./lexer ../src/input.one log
-RUN cat log
+RUN ./lexer ../src/input.one log && cat log
 
 # Running tests
 RUN ./lexer_test

src/ast/ast.h

@@ -14,7 +14,7 @@
 #include <inttypes.h>
 
 #include "../builtins/array.h"
-#include "../lexer/tokenizer/token.h"
+#include "../parser/lexer/token.h"
 
 typedef enum _token_type TokenType;
 typedef struct _location Location;

src/builtins/error.c

@@ -8,7 +8,7 @@
 
  **/
 
-#include <parser/lexer/lexer.h>
+#include "../parser/lexer/lexer.h"
 
 #include "error.h"
 

src/parser/lexer/lexer.c

@@ -12,7 +12,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <builtins/error.h>
+#include "../../builtins/error.h"
 
 #ifdef _ONE_TEST_
 #define debug_lexer(format, args...)   \
@@ -617,3 +617,49 @@ void lexer_free()
 	// free(lexer.start);
 	// free(lexer.current);
 }
+
+/*
+ * @function: lexer_trace
+ * @description: Log and trace items of tokens
+ * @arguments: FILE* file_out, char* data, Token** tokens
+ * @return: nothing, void
+ */
+void lexer_trace(FILE* file_out, const char* data, Token** tokens)
+{
+	debug_lexer("lexer_trace");
+
+	while (tokens != NULL && *tokens != NULL)
+		if (tokens == NULL) return;
+
+	while (*tokens != NULL)
+	{
+		Token* t = *tokens;
+		printf("sizeof(t) = %zu\n", sizeof t);
+		printf("-->1\n");
+		printf("-->%s\n", t == NULL ? "y" : "n");
+		// printf("sizeof(t) = %zu\n", sizeof t);
+		// printf("-->1\n");
+		// printf("-->%s\n", t == NULL ? "y" : "n");
+		char* t_name = token_name(t->type);
+		printf("-->2\n");
+		// printf("-->2\n");
+		bool has1 = file_convert_index_to_rc(data, t->pos.index, &t->pos.line, &t->pos.column);
+		printf("-->3\n");
+		// printf("-->3\n");
+		bool has2 = file_convert_index_to_rc(data, t->pos_end.index, &t->pos_end.line, &t->pos_end.column);
+		printf("-->4\n");
+		// printf("-->4\n");
+
+		fprintf(file_out, "[%zu:%zu] [%zu:%zu - %zu:%zu] %s", t->pos.tokens, t->length, t->pos.line, t->pos.column, t->pos_end.line, t->pos_end.column, t_name);
+		printf("-->5\n");
+		// printf("-->5\n");
+
+		if (t->value != NULL)
+		{
+			fprintf(file_out, ": \"%s\"", t->value);
+		}
+		fprintf(file_out, "\n");
+		if (t->type == TOKEN_EOF) break;
+		tokens++;
+	}
+}

src/parser/lexer/main.c

@@ -11,7 +11,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-#include <builtins/file.h>
+#include "../../builtins/file.h"
 
 #include "lexer.h"
 extern Token* current;

src/parser/lexer/token.c

@@ -10,7 +10,8 @@
 
 #include "token.h"
 #include "lexer.h"
-#include <builtins/error.h>
+#include "../../builtins/array.h"
+#include "../../builtins/error.h"
 
 #ifdef _ONE_TEST_
 #define debug_token(format, args...)     \
@@ -587,3 +588,49 @@ bool token_is_skip(TokenType type)
 		 type == TOKEN_SKIP_COMMENT_SINGLE ||
 		 type == TOKEN_SKIP_COMMENT_MULTI);
 }
+
+/*
+ * @function: tokenizer_string
+ * @description: Create a array of tokens from a one program source-code char*
+ * @inputs: char* of a One program source-code
+ * @return: Array of Token
+ */
+Token** tokenizer_string(char* data)
+{
+	debug_token("tokenizer_string");
+	debug_token("tokenizer_string: %s", data);
+
+	lexer_init(data);
+
+	Array tokens;
+	array_init(&tokens);
+
+	size_t i;
+	Token* t;
+	for (;;)
+	{
+		t = lexer_scan();
+		// printf("==>%s\n", token_name(t->type));
+		array_push(&tokens, t);
+		// debug_lexer("parser_scan: print_token %s", token_name(t->type));
+		if (t->type == TOKEN_ERROR)
+		{
+			printf("Error: %s\n", t->value);
+			break;
+		}
+		else if (t->type == TOKEN_EOF)
+			break;
+	}
+
+	debug_token("tokenizer_string: count of tokens is %d", tokens.count);
+	debug_token("tokenizer_string: size of tokens is %d", tokens.size);
+
+	// while (*data != '\0')
+	// {
+	// 	Token* t = token_make_value(TOKEN_VALUE_IDENTIFIER, (char*){data});
+	// 	array_push(&tokens, t);
+	// 	data++;
+	// }
+
+	return (Token**)tokens.data;
+}

src/parser/lexer/token.h

@@ -407,4 +407,12 @@ size_t token_utf8_string_length(char* s);
  */
 bool token_is_skip(TokenType type);
 
+/*
+ * @function: tokenizer_string
+ * @description: Create a array of tokens from a one program source-code char*
+ * @inputs: char* of a One program source-code
+ * @return: Array of Token
+ */
+Token** tokenizer_string(char* data);
+
 #endif // _ONE_TOKENIZER_TOKEN_H_

src/parser/parser.h

@@ -17,7 +17,7 @@
 
 #include "parser_token.h"
 
-#include <builtins/array.h>
+#include "../builtins/array.h"
 
 typedef struct
 {

test/test.h

@@ -23,7 +23,7 @@
 
 #include "../src/builtins/file.h"
 
-#include "../src/lexer/lexer.h"
+#include "../src/parser/lexer/lexer.h"
 #include "../src/parser/parser.h"
 
 typedef struct