General code cleanup, plus more comments

Author: Others

Cargo.toml

@@ -6,7 +6,7 @@ authors = ["Gregor Peach <[email protected]>"]
 [dependencies]
 llvm-alt = { git = "https://github.com/Others/llvm-rs.git"}
 structopt = "0.2"
-wasmparser = "0.16.1"
+wasmparser = "0.22"
 
 [profile.release]
 debug = true

code_benches/dummy.c

@@ -1,3 +1,9 @@
+// !!! HACK ALERT !!!
+// We need the __init_libc symbol in the output executable (so the runtime can initialize libc)
+// We can't directly export it since it's in a linked library
+// Thus we export a dummy function that uses it, forcing it to be included
+
+
 #define IMPORT __attribute__ ((visibility ("default")))
 #define EXPORT __attribute__ ((visibility ("default")))
 

code_benches/run.py

@@ -1,26 +1,35 @@
-from collections import namedtuple
 import os
 import subprocess as sp
 import sys
 import timeit
 
+# Absolute path to the `code_benches` directory
 BENCH_ROOT = os.getcwd()
+# Absolute path to the `silverfish` directory
 ROOT_PATH = os.path.dirname(BENCH_ROOT)
+
 RUNTIME_PATH = ROOT_PATH + "/runtime"
 SILVERFISH_PATH = ROOT_PATH + "/target/release/silverfish"
 WASMCEPTION_PATH = ROOT_PATH + "/wasmception"
 
+# Our special WASM clang is under this wasmception path
 WASM_CLANG = WASMCEPTION_PATH + "/dist/bin/clang"
+# These flags are all somewhat important -- see @Others for more information
 WASM_LINKER_FLAGS = "-Wl,--allow-undefined,-z,stack-size={stack_size},--no-threads,--stack-first,--no-entry,--export-all,--export=main,--export=dummy"
+# Point WASM to our custom libc
 WASM_SYSROOT_FLAGS = "--sysroot={}/sysroot".format(WASMCEPTION_PATH)
 WASM_FLAGS = WASM_LINKER_FLAGS + " --target=wasm32-unknown-unknown-wasm -nostartfiles -O3 -flto " + WASM_SYSROOT_FLAGS
 
+# What is the machine we're running on like?
 IS_64_BIT = sys.maxsize > 2**32
 IS_X86 = '86' in os.uname()[-1]
 IS_32_BIT_X86 = (not IS_64_BIT) and IS_X86
 
+# TODO: Add an option to output a CSV instead of human readable output
+
+# How many times should we run our benchmarks
 RUN_COUNT = 10
-ENABLE_DEBUG_SYMBOLS = False
+ENABLE_DEBUG_SYMBOLS = True
 
 
 # FIXME: Mibench runs many of these programs multiple times, which is probably worth replicating
@@ -73,7 +82,7 @@ def __str__(self):
 ]
 
 
-# Now some helper methods for compiling code
+# Compile the C code in `program`'s directory into a native executable
 def compile_to_executable(program):
     opt = "-O3"
     if program.do_lto:
@@ -83,21 +92,25 @@ def compile_to_executable(program):
     sp.check_call("clang {} -lm {} *.c -o bin/{}".format(program.custom_arguments, opt, program.name), shell=True, cwd=program.name)
 
 
+# Compile the C code in `program`'s directory into WASM
 def compile_to_wasm(program):
     flags = WASM_FLAGS.format(stack_size=program.stack_size)
     command = "{clang} {flags} {args} -O3 -flto ../dummy.c *.c -o bin/{pname}.wasm" \
         .format(clang=WASM_CLANG, flags=flags, args=program.custom_arguments, pname=program.name)
     sp.check_call(command, shell=True, cwd=program.name)
 
 
+# Compile the WASM in `program`'s directory into llvm bytecode
 def compile_wasm_to_bc(program):
     command = "{silverfish} bin/{pname}.wasm -o bin/{pname}.bc".format(silverfish=SILVERFISH_PATH, pname=program.name)
     sp.check_call(command, shell=True, cwd=program.name)
-    # Compile a second version with runtime globlas
+    # Compile a second version with runtime globals
+    # FIXME: Runtime globals were a failed experiment -- evaluate removing all traces of it
     command = "{silverfish} -i --runtime_globals bin/{pname}.wasm -o bin/{pname}_rg.bc".format(silverfish=SILVERFISH_PATH, pname=program.name)
     sp.check_call(command, shell=True, cwd=program.name)
 
 
+# Compile the WASM in `program`'s directory into llvm bytecode
 def compile_wasm_to_executable(program, exe_postfix, memory_impl, runtime_globals=False):
     bc_file = "bin/{pname}.bc".format(pname=program.name)
     if runtime_globals:
@@ -112,18 +125,24 @@ def compile_wasm_to_executable(program, exe_postfix, memory_impl, runtime_global
     sp.check_call(command, shell=True, cwd=program.name)
 
 
+# Execute executable `p` with arguments `args` in directory `dir`
 def execute(p, args, dir):
     command = p + " " + args
     sp.check_call(command, shell=True, stdout=sp.DEVNULL, stderr=sp.DEVNULL, cwd=dir)
 
 
+# Benchmark the given program's executable
+#   p = the program
+#   exe_postfix = what postfix we gave the executable in `compile_wasm_to_executable`
+#   name = the human readable name for this version of the executable
 def bench(p, exe_postfix, name):
     print("Testing {} {}".format(p, name))
     path = "{broot}/{pname}/bin/{pname}{pf}".format(broot=BENCH_ROOT, pname=p.name, pf=exe_postfix)
     command = "execute('{path}', '{args}', '{dir}')".format(path=path, args=' '.join(map(str, p.parameters)), dir=p.name)
     return min(timeit.repeat(command, 'from __main__ import execute', number=1, repeat=RUN_COUNT))
 
 
+# Output a run's execution time, telling us how much faster or slower it is
 def output_run(base_time, execution_time):
     base_time = round(base_time, 4)
     execution_time = round(execution_time, 4)
@@ -133,6 +152,7 @@ def output_run(base_time, execution_time):
         print("{:.4f} ({:.2f}% faster)".format(execution_time, ((base_time - execution_time) / base_time) * 100))
 
 
+# Compile all our programs
 for i, p in enumerate(programs):
     print("Compiling {} {}/{}".format(p.name, i + 1, len(programs)))
 
@@ -151,6 +171,7 @@ def output_run(base_time, execution_time):
 
 print()
 
+# Benchmark and output timing info for each of our programs
 for p in programs:
     base_speed = bench(p, "", "native")
     print("{:.4f}".format(base_speed))

example_code/binarytrees.wasm

@@ -1,9 +1,10 @@
 #include "../runtime.h"
 
+// What should we tell the child program its UID and GID are?
 #define UID 0xFF
 #define GID 0xFE
 
-// Elf auxilary vector values
+// Elf auxilary vector values (see google for what those are)
 #define AT_NULL		0
 #define AT_IGNORE	1
 #define AT_EXECFD	2
@@ -24,10 +25,14 @@
 #define AT_BASE_PLATFORM 24
 #define AT_RANDOM	25
 
+// The symbol the binary gives us to init libc
 void wasmf___init_libc(i32 envp, i32 pn);
 
+// offset = a WASM ptr to memory the runtime can use
 void stub_init(i32 offset) {
+    // What program name will we put in the auxiliary vectors
     char program_name[] = "wasm_program";
+    // Copy the program name into WASM accessible memory
     i32 program_name_offset = offset;
     strcpy(get_memory_ptr_for_runtime(offset, sizeof(program_name)), program_name);
     offset += sizeof(program_name);
@@ -37,7 +42,7 @@ void stub_init(i32 offset) {
     i32 env_vec[] = {
         // Env variables would live here, but we don't supply any
         0,
-        // We supply nessesary
+        // We supply only the bare minimum AUX vectors
         AT_PAGESZ,
         WASM_PAGE_SIZE,
         AT_UID,
@@ -62,7 +67,7 @@ void stub_init(i32 offset) {
     switch_into_runtime();
 }
 
-// Syscall stuff
+// Emulated syscall implementations
 
 // We define our own syscall numbers, because WASM uses x86_64 values even on systems that are not x86_64
 #define SYS_READ 0
@@ -90,8 +95,7 @@ i32 wasm_write(i32 fd, i32 buf_offset, i32 buf_size) {
 
 #define SYS_OPEN 2
 i32 wasm_open(i32 path_off, i32 flags, i32 mode) {
-    // TODO: Handle string being bad
-    char* path = get_memory_ptr_void(path_off, 0);
+    char* path = get_memory_string(path_off);
     i32 res = (i32) open(path, flags, mode);
 
     if (res == -1) {
@@ -111,6 +115,8 @@ i32 wasm_close(i32 fd) {
     return res;
 }
 
+#define SYS_FSTAT 5
+// What the wasm stat structure looks like
 struct wasm_stat {
 	i64 st_dev;
 	u64 st_ino;
@@ -131,6 +137,7 @@ struct wasm_stat {
 	i32 __pad1[3];
 };
 
+// What the OSX stat structure looks like:
 //     struct stat { /* when _DARWIN_FEATURE_64_BIT_INODE is NOT defined */
 //         dev_t    st_dev;    /* device inode resides on */
 //         ino_t    st_ino;    /* inode's number */
@@ -150,7 +157,6 @@ struct wasm_stat {
 //     };
 
 
-#define SYS_FSTAT 5
 i32 wasm_fstat(i32 filedes, i32 stat_offset) {
     struct wasm_stat* stat_ptr = get_memory_ptr_void(stat_offset, sizeof(struct wasm_stat));
 
@@ -217,16 +223,13 @@ u32 wasm_mmap(i32 addr, i32 len, i32 prot, i32 flags, i32 fd, i32 offset) {
 
 #define SYS_RT_SIGACTION 13
 
-
 #define SYS_RT_SIGPROGMASK 14
-//i32 wasm_sigprocmask(i32 how, i32 set_ptr, i32 oset_ptr) {
-//    return 0;
-//}
 
 #define SYS_IOCTL 16
 i32 wasm_ioctl(i32 fd, i32 request, i32 data_offet) {
-	/* musl libc does some ioctls to stdout, so just allow these to silently go through */
-	return 0;
+    // musl libc does some ioctls to stdout, so just allow these to silently go through
+    // FIXME: The above is idiotic
+    return 0;
 }
 
 #define SYS_READV 19
@@ -492,24 +495,3 @@ INLINE double env_cos(double d) {
     return cos(d);
 }
 
-
-// gdbm code
-//int gdbm_file = 0;
-//GDBM_FILE gdbm_files[100];
-//
-//i32 env_gdbm_open(i32 name_off, i32 block_size, i32 read_write, i32 mode, i32 fatal_func) {
-//    char* name = get_memory_ptr_void(name, 0);
-//    GDBM_FILE f = gdbm_open(name, block_size, read_write, mode, NULL);
-//    gdbm_files[gdbm_file] = f;
-//    gdbm_file++;
-//    return (i32) gdbm_file;
-//}
-//
-//i32 env_gdbm_close(i32 gf) {
-//
-//}
-//
-//
-//void env_gdbm_fetch(i32 a, i32 b, i32 c) {
-//}
-//