Add application based tests

scopeInfinity · scopeInfinity · commit 5405e650f0c6 · 2021-08-22T00:45:59.000+01:00
diff --git a/.gitignore b/.gitignore
@@ -3,4 +3,6 @@
 local_notes/
 build/
 src_test/
-build_test/
+build_test/
+__pycache__/
+.vscode/
diff --git a/before_install.sh b/before_install.sh
@@ -1,3 +1,5 @@
 #!/bin/bash
 apt-get update
 apt install -y make nasm gcc cpulimit qemu-system-x86 gocr expect gdb graphicsmagick-imagemagick-compat
+
+python3 -m pip install -r requirements.txt
diff --git a/emulator/Makefile.mk b/emulator/Makefile.mk
@@ -1,6 +1,7 @@
 QEMU_SHUT_FLAGS= -no-shutdown -no-reboot
 QEMU_EXTRA_FLAGS=
 QEMU_GDB_PORT=9000
+QEMU_MONITOR_PORT=55555
 GDB_EX='echo Use GDB_EX="<command>" to execute command on connect.\n'
 
 qemu: $(image_vmdk)
@@ -9,6 +10,9 @@ qemu: $(image_vmdk)
 qemu_vvv: $(image_vmdk)
 	qemu-system-x86_64 -smp 1 -m 128M -hda $< $(QEMU_SHUT_FLAGS) $(QEMU_EXTRA_FLAGS) -d  cpu,exec,in_asm
 
+qemu_monitor: $(image_vmdk)
+	qemu-system-x86_64 -smp 1 -m 128M -hda $< $(QEMU_SHUT_FLAGS) $(QEMU_EXTRA_FLAGS) -monitor telnet:127.0.0.1:$(QEMU_MONITOR_PORT),server,nowait
+
 qemu_debug: $(image_vmdk)
 	qemu-system-x86_64 -S -gdb tcp::$(QEMU_GDB_PORT) -smp 1 -m 128M -hda $< $(QEMU_SHUT_FLAGS) -d  cpu,exec,in_asm
 
diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1 @@
+pexpect
diff --git a/tests/__init__.py b/tests/__init__.py
diff --git a/tests/app/cat_test.sh b/tests/app/cat_test.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# Test is supposed to be sourced from shell_test.sh
+
+python3 -m tests.qemu.monitor -p ${MONITOR_PORT:?} -sc run cat
+
+test_create_screen_dump
+test_screen_content $LINENO "Experimental OS"
+test_screen_content $LINENO "The project is still under development"
diff --git a/tests/app/ls_test.sh b/tests/app/ls_test.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Test is supposed to be sourced from shell_test.sh
+
+python3 -m tests.qemu.monitor -p ${MONITOR_PORT:?} -sc run ls
+
+test_create_screen_dump
+test_screen_content $LINENO "cat"
+test_screen_content $LINENO "calc"
+test_screen_content $LINENO "ls"
+test_screen_content $LINENO "tictactoe"
diff --git a/tests/app/shell_test.sh b/tests/app/shell_test.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Test is supposed to be sourced from fuzzy_app_test.sh
+
+test_screen_content $LINENO "Simple Shell"
+
+# Test applications
+source tests/app/ls_test.sh
+source tests/app/tictactoe_test.sh
+source tests/app/cat_test.sh
diff --git a/tests/app/tictactoe_test.sh b/tests/app/tictactoe_test.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Test is supposed to be sourced from shell_test.sh
+
+python3 -m tests.qemu.monitor -p ${MONITOR_PORT:?} -sc run tictactoe
+
+test_create_screen_dump
+test_screen_content $LINENO "TicTacToe"
+test_screen_content $LINENO "Q to quit game"
+
+python3 -m tests.qemu.monitor -p ${MONITOR_PORT:?} -sc 'q'
diff --git a/tests/bootloader_stage1_test.sh b/tests/bootloader_stage1_test.sh
@@ -14,5 +14,6 @@ set -x
 test_screen_content $LINENO "Bootloader: Stage 1"
 test_screen_content $LINENO "Stage 2 Loaded: `build_8hexbyte bootloader/stage2`"
 
+python -m tests.qemu.monitor --quit
 wait ${QEMU_PID:?}
 echo "$0 passed!!!"
diff --git a/tests/bootloader_stage2_test.sh b/tests/bootloader_stage2_test.sh
@@ -15,5 +15,6 @@ test_screen_content $LINENO "Static library loaded at 0x00007E00: `build_8hexbyt
 test_screen_content $LINENO "Kernel loaded at 0x[0-9A-Fa-f]\+: `build_8hexbyte kernel/core`"
 test_screen_content $LINENO "Loading GDT Table and entering protected mode"
 
+python -m tests.qemu.monitor --quit
 wait ${QEMU_PID:?}
 echo "$0 passed!!!"
diff --git a/tests/fuzzy_app_test.sh b/tests/fuzzy_app_test.sh
@@ -2,14 +2,14 @@
 
 source tests/lib.sh
 
-TEST_INJECT_WORD="__TEST_INJECT_APP_TTT_ENTRY__"
+TEST_INJECT_WORD="${MAGIC_WORD_NO_TEST_INJECT:?}"
 TEST_MAGIC_WANT="${MAGIC_WORD_SLEEP:?}"
 
 os_test_up "${TEST_MAGIC_WANT:?}" "${TEST_INJECT_WORD:?}" || exit -1
 
-# Test
-set -e
-test_screen_content $LINENO "TicTacToe"
+# Assumes Fuzzy OS starts with Shell
+source tests/app/shell_test.sh
 
+python -m tests.qemu.monitor --quit
 wait ${QEMU_PID:?}
 echo "$0 passed!!!"
diff --git a/tests/kernel_core_entry_test.sh b/tests/kernel_core_entry_test.sh
@@ -12,5 +12,7 @@ set -e
 test_screen_content $LINENO "Initializing Kernel"
 test_screen_content $LINENO "Registering syscalls"
 
+
+python -m tests.qemu.monitor --quit
 wait ${QEMU_PID:?}
 echo "$0 passed!!!"
diff --git a/tests/lib.sh b/tests/lib.sh
@@ -1,14 +1,16 @@
 #!/bin/bash
 
+set -e
+
 SRC_DIR="src/"
 SRC_TEST_DIR="src_test/"
 BUILD_TEST_DIR="build_test"
 MONITOR_PORT=55555
 QEMU_SCREENSHOT="/tmp/$(basename $0 .sh).ppm"
 QEMU_SCREENSHOT_ARTIFACT="${QEMU_SCREENSHOT%.ppm}.png"
-INIT_APPNAME="tictactoe"
 
 MAGIC_WORD_SLEEP="##SLEEP-10s##"
+MAGIC_WORD_NO_TEST_INJECT="TEST-INJECT-WORD-NOT-NEEDED"
 
 ##########################################
 # Raise error.
@@ -64,6 +66,27 @@ function test_screen_content() {
     err "${1:?}" "Test Failed! Couldn't find '${2:?}' on screen."
 }
 
+##########################################
+# Test Helper: Create QEMU screen dump
+# Globals:
+#   QEMU_SCREENSHOT
+#   QEMU_SCREENSHOT_ARTIFACT
+# Output:
+#   raises error or screen dump in text.
+##########################################
+function test_create_screen_dump() {
+    python -m tests.qemu.monitor -qc screendump ${QEMU_SCREENSHOT:?}
+
+    if [ ! -f ${QEMU_SCREENSHOT:?} ]; then
+        err -1 "[create_screen_dump] failed to create qemu screen dump"
+    fi
+
+    convert ${QEMU_SCREENSHOT:?} ${QEMU_SCREENSHOT_ARTIFACT:?}
+
+    SCREEN_CONTENT="$(gocr -i ${QEMU_SCREENSHOT:?})"
+    echo "Screen Content: '${SCREEN_CONTENT:?}'"
+}
+
 ##########################################
 # Activate Code for testing within source.
 # Arguments:
@@ -92,6 +115,10 @@ function sync_to_src_test() {
     rm -r "${SRC_TEST_DIR:?}"
     cp -r "${SRC_DIR:?}" "${SRC_TEST_DIR:?}"
 
+    if [[ "${1:?}" == "${MAGIC_WORD_NO_TEST_INJECT}" ]]; then
+        return
+    fi
+
     find "${SRC_TEST_DIR:?}" -iname '*.asm' -exec bash -c 'inject_test_code_asm "$0" "$1"' {} "$1" \;
     find "${SRC_TEST_DIR:?}" -iname '*.c' -exec bash -c 'inject_test_code_c "$0" "$1"' {} "$1" \;
 }
@@ -102,7 +129,6 @@ function sync_to_src_test() {
 #   COMMAND_OUTPUT
 #   SCREEN_CONTENT
 #   QEMU_PID
-#   INIT_APPNAME
 # Arguments:
 #   Magic Word
 #   Inject Keyword
@@ -120,7 +146,6 @@ function os_test_up() {
     # Turn up QEMU in background
     make clean BUILD_DIR="${BUILD_TEST_DIR:?}" \
         && make qemu \
-        INIT_APPNAME="${INIT_APPNAME:?}" \
         SRC_DIR="${SRC_TEST_DIR:?}" \
         BUILD_DIR="${BUILD_TEST_DIR:?}" \
         QEMU_SHUT_FLAGS="" \
@@ -155,18 +180,8 @@ function os_test_up() {
     fi
 
     sleep 1s
-    ./tests/qemu_monitor_expect.sh ${MONITOR_PORT:?} "screendump ${QEMU_SCREENSHOT:?}"
-    ./tests/qemu_monitor_expect.sh ${MONITOR_PORT:?} "quit"
-
-    if [ ! -f ${QEMU_SCREENSHOT:?} ]; then
-        echo "Magic word found but no screenshot found! :( " >&2
-        return -1
-    fi
-
-    convert ${QEMU_SCREENSHOT:?} ${QEMU_SCREENSHOT_ARTIFACT:?}
 
-    SCREEN_CONTENT="$(gocr -i ${QEMU_SCREENSHOT:?})"
-    echo "Screen Content: '${SCREEN_CONTENT:?}'"
+    test_create_screen_dump
 
     echo "os_up done."
     return 0
diff --git a/tests/qemu/__init__.py b/tests/qemu/__init__.py
diff --git a/tests/qemu/args.py b/tests/qemu/args.py
@@ -0,0 +1,23 @@
+import argparse
+
+def get_args():
+    parser = argparse.ArgumentParser(description="Fuzzy OS QEMU monitor client")
+    parser.add_argument("-p", "--port",
+                        type=int,
+                        default=55555,
+                        help="qemu monitor port")
+    parser.add_argument('-qc', '--qemu-cmd',
+                        metavar="qemu-token",
+                        type=str,
+                        nargs='+',
+                        help="command to execute in qemu monitor")
+    parser.add_argument('-sc', '--fuzzy-shell-cmd',
+                        metavar="fuzzy-token",
+                        type=str,
+                        nargs='+',
+                        help="command to execute in Fuzzy OS shell")
+    parser.add_argument('-q', '--quit',
+                        action='store_true',
+                        help="kill qemu before leaving")
+    args = parser.parse_args()
+    return args
diff --git a/tests/qemu/key.py b/tests/qemu/key.py
@@ -0,0 +1,13 @@
+QEMU_CODES = {
+    " ": "spc",
+    "\n": "ret"
+}
+
+def char_to_key(char):
+    if char in QEMU_CODES:
+        return QEMU_CODES[char]
+
+    if isinstance(char, str) and len(char) == 1 and char.isalnum():
+        # single isalnum char
+        return char
+    raise Exception(f"char_to_key can't handle '{char}'")
diff --git a/tests/qemu/monitor.py b/tests/qemu/monitor.py
@@ -0,0 +1,76 @@
+import logging
+import pexpect
+import time
+
+from tests.qemu import args
+from tests.qemu import key
+
+logger = logging.getLogger(__name__)
+
+QEMU_PROMPT = "(qemu)"
+
+
+class Monitor:
+
+    def __init__(self, port):
+        logger.info("Connecting to qemu, waiting for prompt")
+        self.process = pexpect.spawn(f"telnet 127.0.0.1 {port}")
+        self.wait_for_prompt()
+
+    def wait_for_prompt(self):
+        try:
+            self.process.expect(QEMU_PROMPT, timeout=2)
+        except pexpect.TIMEOUT as e:
+            logger.error("[pexpect] Qemu prompt wasn't seen within timeout")
+            raise e
+
+    def send_qemu_command(self, cmd, wait_prompt=True):
+        self.process.sendline(cmd)
+        if wait_prompt:
+            self.wait_for_prompt()
+
+    def send_key(self, key):
+        """Send a key to qemu."""
+        self.process.sendline(f"sendkey {key}")
+        self.wait_for_prompt()
+
+    def type_string(self, msg):
+        """Type string in qemu."""
+        logger.info(f"[to_qemu] typing string: {repr(msg)}")
+        for char in msg:
+            self.send_key(key.char_to_key(char))
+
+def handle_qemu_cmd(monitor, args):
+    if not args.qemu_cmd:
+        return
+    logger.info("Handle: fuzzy_shell_cmd")
+    cmd = " ".join(args.qemu_cmd) + "\n"
+    monitor.send_qemu_command(cmd)
+
+def handle_fuzzy_shell_cmd(monitor, args):
+    if not args.fuzzy_shell_cmd:
+        return
+    logger.info("Handle: fuzzy_shell_cmd")
+    cmd = " ".join(args.fuzzy_shell_cmd) + "\n"
+    monitor.type_string(cmd)
+
+def handle_qemu_quit(monitor, args):
+    if not args.quit:
+        return
+    logger.info("Handle: qemu quit")
+    monitor.send_qemu_command("quit\n", wait_prompt=False)
+
+def main():
+    logging.basicConfig(level=logging.INFO)
+    parsed_args = args.get_args()
+
+    monitor = Monitor(parsed_args.port)
+
+    handle_fuzzy_shell_cmd(monitor, parsed_args)
+    handle_qemu_cmd(monitor, parsed_args)
+    handle_qemu_quit(monitor, parsed_args)
+
+    logger.info("graceful exit")
+
+if __name__ == '__main__':
+    main()
diff --git a/tests/run.sh b/tests/run.sh
@@ -2,10 +2,10 @@
 set -e
 
 # Execute Tests
-timeout 2m bash tests/bootloader_stage1_test.sh
-timeout 2m bash tests/bootloader_stage2_test.sh
-timeout 2m bash tests/kernel_core_entry_test.sh
-timeout 2m bash tests/app_ttt_test.sh
+timeout 2m bash tests/bootloader_stage1_test.sh &&
+timeout 2m bash tests/bootloader_stage2_test.sh &&
+timeout 2m bash tests/kernel_core_entry_test.sh &&
+timeout 2m bash tests/fuzzy_app_test.sh
 
 # Done
 echo "All tests passed!"
