Create VERIFY_STACKGUARD to prevent stackoverflow in kernel

Author: scopeInfinity

Makefile

@@ -102,7 +102,7 @@ binaries: $(bt_stage1) $(bt_stage2) $(kernel_core) $(rm_static)
 SECTOR_COUNT_BT_STAGE1 = 1
 SECTOR_COUNT_SHARED_LIBRARY = 1
 SECTOR_COUNT_BT_STAGE2 = 11
-SECTOR_COUNT_KERNEL = 47
+SECTOR_COUNT_KERNEL = 48
 
 SECTOR_START_BT_STAGE1 = 0
 SECTOR_START_SHARED_LIBRARY = $(shell expr $(SECTOR_START_BT_STAGE1) + $(SECTOR_COUNT_BT_STAGE1) )
@@ -141,6 +141,7 @@ include $(SRC_DRIVERS)/pic/Makefile.mk
 
 include $(SRC_DIR)/fs/Makefile.mk
 include $(SRC_DIR)/memmgr/tables/Makefile.mk
+include $(SRC_DIR)/memmgr/stackguard/Makefile.mk
 
 include $(SRC_LIB)/app/Makefile.mk
 include $(SRC_LIB_DS)/Makefile.mk

include/fuzzy/memmgr/stackguard/stackguard.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#define __MACRO_TO_STRING_INTERNAL(x)   #x
+#define MACRO_TO_STRING(x)              __MACRO_TO_STRING_INTERNAL(x)
+#define __STR__LINE__                   MACRO_TO_STRING(__LINE__)
+
+#define VERIFY_STACKGUARD() (verify_stack_guard(__FILE__ "[" __STR__LINE__ "]; verify_stack_guard failed"))
+
+void verify_stack_guard(char err_message[]);

src/kernel/Makefile.mk

@@ -13,7 +13,9 @@ $(SELF_BUILD_DIR)/%_asm.o: $(SELF_SRC_DIR)/%.asm
 	mkdir -p $(dir $@)
 	$(NASM) -o $@ -i $(SRC_REALMODE)/ $<
 
-$(kernel_core).elf: $(SELF_BUILD_DIR)/core_asm.o $(SELF_BUILD_DIR)/panic_asm.o $(SELF_BUILD_ALL_C) \
+$(kernel_core).elf: $(SELF_BUILD_DIR)/core_asm.o \
+		$(SELF_BUILD_DIR)/panic_asm.o \
+		$(SELF_BUILD_ALL_C) \
 		$(BUILD_KERNEL)/interrupts/libinterrupts \
 		$(BUILD_KERNEL)/syscall/libsyscall \
 		$(BUILD_KERNEL)/process/libprocess \
@@ -26,7 +28,9 @@ $(kernel_core).elf: $(SELF_BUILD_DIR)/core_asm.o $(SELF_BUILD_DIR)/panic_asm.o $
 		$(BUILD_LIB_DS)/libds \
 		$(BUILD_DRIVERS)/disk/libdisk \
 		$(BUILD_DIR)/real_mode/librealmodeclient \
-		$(BUILD_USR_LIB)/libfuzzyc
+		$(BUILD_USR_LIB)/libfuzzyc \
+		$(BUILD_DIR)/memmgr/stackguard/libstackguard # stackguard must be the last one
+
 	mkdir -p $(dir $@)
 	$(KERNEL_LD) -o $@ $^
 

src/kernel/core.c

@@ -3,6 +3,7 @@
 #include <fuzzy/kernel/interrupts/interrupts.h>
 #include <fuzzy/kernel/interrupts/timer.h>
 #include <fuzzy/kernel/process/process.h>
+#include <fuzzy/memmgr/stackguard/stackguard.h>
 
 #include <string.h>
 #include <process.h>
@@ -50,6 +51,7 @@ void kernel_core_entry() {
 
     clrscr();
 
+    VERIFY_STACKGUARD();
     int init_pid = spawnl(INIT_APPNAME, INIT_APPNAME, NULL);
     print_log("init process got created: %d", init_pid);
 

src/memmgr/stackguard/Makefile.mk

@@ -0,0 +1,10 @@
+$(SELF_BUILD_DIR)/%.o: $(SELF_SRC_DIR)/%.c $(BUILD_USR_INCLUDE_ALL)
+	mkdir -p $(dir $@)
+	$(KERNEL_CC) -c -o $@ $<
+
+$(SELF_BUILD_DIR)/%_asm.o: $(SELF_SRC_DIR)/%.asm
+	mkdir -p $(dir $@)
+	nasm -o $@ -f elf32 $<
+
+$(SELF_BUILD_DIR)/libstackguard: $(SELF_BUILD_ALL_C) $(SELF_BUILD_ALL_ASM)
+	ar rc $@ $^

src/memmgr/stackguard/stackguard.asm

@@ -0,0 +1,26 @@
+; Corresponding object file should be linked at the tail of the
+; instructions.
+; Assumes .data gets linked after .text
+
+[BITS 32]
+
+global check_stack_guard
+
+; it would be nice if magic number is present at head of
+; instruction but this should be enough for now.
+STACK_GUARD_MAGIC_NUMBER    EQU     0x4C10A5C7
+
+[SECTION .text]
+    check_stack_guard:
+        ; return 0 is stack is good
+        push ebp
+        mov ebp, esp
+
+        mov eax, [stack_guard_pointer]
+        xor eax, STACK_GUARD_MAGIC_NUMBER
+
+        pop ebp
+        ret
+
+[SECTION .data]
+    stack_guard_pointer     dd      STACK_GUARD_MAGIC_NUMBER

src/memmgr/stackguard/stackguard.c

@@ -0,0 +1,11 @@
+#include <fuzzy/kernel/panic.h>
+#include <fuzzy/memmgr/stackguard/stackguard.h>
+
+extern int check_stack_guard();
+
+void verify_stack_guard(char err_message[]) {
+    int err = check_stack_guard();
+    if (err != 0) {
+        PANIC(0, err_message);
+    }
+}