Get wasm-pack supposedly working (#16)

Author: DavisVaughan

Cargo.toml

@@ -109,3 +109,10 @@ unnested_or_patterns = "warn"
 unreadable_literal = "warn"
 verbose_bit_mask = "warn"
 verbose_file_reads = "warn"
+
+# Profiles must be set at the workspace level
+[profile.dev]
+# Tell `rustc` to optimize for small code size to
+# work around "too many locals" error from wasm-pack
+# https://github.com/wasm-bindgen/wasm-bindgen/issues/3451#issuecomment-1562982835
+opt-level = "s"

crates/tree-sitter-qmd/tree-sitter-markdown/src/scanner.c

@@ -2,7 +2,6 @@
 #include <assert.h>
 #include <ctype.h>
 #include <string.h>
-#include <wchar.h>
 #include <wctype.h>
 
 // For explanation of the tokens see grammar.js
@@ -200,7 +199,7 @@ static size_t roundup_32(size_t x) {
 }
 
 typedef struct {
-    unsigned own_size; 
+    unsigned own_size;
     // Size of the serialized state of the scanner.
     // This is used to determine if we're too close to hitting
     // tree-sitter's maximum serialized size limit of 1024 bytes,
@@ -452,7 +451,7 @@ static bool parse_fenced_div_marker(Scanner *s, TSLexer *lexer,
     //
     // otherwise, it can only be a valid marker for the end of a fenced div
 
-    while (!lexer->eof(lexer) && 
+    while (!lexer->eof(lexer) &&
         (lexer->lookahead == ' ' || lexer->lookahead == '\t')) {
         advance(s, lexer);
     }
@@ -831,7 +830,7 @@ static bool parse_ordered_list_marker(Scanner *s, TSLexer *lexer,
         size_t digits = 1;
         bool dont_interrupt = lexer->lookahead != '1';
         advance(s, lexer);
-        while (isdigit(lexer->lookahead)) {
+        while (iswdigit(lexer->lookahead)) {
             dont_interrupt = true;
             digits++;
             advance(s, lexer);
@@ -1145,7 +1144,8 @@ static bool parse_html_block(Scanner *s, TSLexer *lexer,
         if (next_symbol_valid) {
             // try block 1 names
             for (size_t i = 0; i < NUM_HTML_TAG_NAMES_RULE_1; i++) {
-                if (strcmp(name, HTML_TAG_NAMES_RULE_1[i]) == 0) {
+                // FIXME: I'm guessing on the size here
+                if (strncmp(name, HTML_TAG_NAMES_RULE_1[i], name_length) == 0) {
                     if (starting_slash) {
                         if (valid_symbols[HTML_BLOCK_1_END]) {
                             lexer->result_symbol = HTML_BLOCK_1_END;
@@ -1174,7 +1174,8 @@ static bool parse_html_block(Scanner *s, TSLexer *lexer,
         if (next_symbol_valid || tag_closed) {
             // try block 2 names
             for (size_t i = 0; i < NUM_HTML_TAG_NAMES_RULE_7; i++) {
-                if (strcmp(name, HTML_TAG_NAMES_RULE_7[i]) == 0 &&
+                // FIXME: I'm guessing on the size here
+                if (strncmp(name, HTML_TAG_NAMES_RULE_7[i], name_length) == 0 &&
                     valid_symbols[HTML_BLOCK_6_START]) {
                     lexer->result_symbol = HTML_BLOCK_6_START;
                     if (!s->simulate) {

crates/wasm-qmd-parser/Cargo.toml

@@ -22,7 +22,3 @@ console_error_panic_hook = { version = "0.1.7", optional = true }
 
 [dev-dependencies]
 wasm-bindgen-test = "0.3.34"
-
-[profile.release]
-# Tell `rustc` to optimize for small code size.
-opt-level = "s"

crates/wasm-qmd-parser/src/c_shim.rs

@@ -0,0 +1,237 @@
+use std::{
+    alloc::{self, Layout},
+    ffi::{c_char, c_int, c_void},
+    mem::align_of,
+    ptr,
+};
+
+/* -------------------------------- stdlib.h -------------------------------- */
+
+#[no_mangle]
+pub unsafe extern "C" fn abort() {
+    panic!("Aborted from C");
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn malloc(size: usize) -> *mut c_void {
+    if size == 0 {
+        return ptr::null_mut();
+    }
+
+    let (layout, offset_to_data) = layout_for_size_prepended(size);
+    let buf = alloc::alloc(layout);
+    store_layout(buf, layout, offset_to_data)
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn calloc(count: usize, size: usize) -> *mut c_void {
+    if count == 0 || size == 0 {
+        return ptr::null_mut();
+    }
+
+    let (layout, offset_to_data) = layout_for_size_prepended(size * count);
+    let buf = alloc::alloc_zeroed(layout);
+    store_layout(buf, layout, offset_to_data)
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn realloc(buf: *mut c_void, new_size: usize) -> *mut c_void {
+    if buf.is_null() {
+        malloc(new_size)
+    } else if new_size == 0 {
+        free(buf);
+        ptr::null_mut()
+    } else {
+        let (old_buf, old_layout) = retrieve_layout(buf);
+        let (new_layout, offset_to_data) = layout_for_size_prepended(new_size);
+        let new_buf = alloc::realloc(old_buf, old_layout, new_layout.size());
+        store_layout(new_buf, new_layout, offset_to_data)
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn free(buf: *mut c_void) {
+    if buf.is_null() {
+        return;
+    }
+    let (buf, layout) = retrieve_layout(buf);
+    alloc::dealloc(buf, layout);
+}
+
+// In all these allocations, we store the layout before the data for later retrieval.
+// This is because we need to know the layout when deallocating the memory.
+// Here are some helper methods for that:
+
+/// Given a pointer to the data, retrieve the layout and the pointer to the layout.
+unsafe fn retrieve_layout(buf: *mut c_void) -> (*mut u8, Layout) {
+    let (_, layout_offset) = Layout::new::<Layout>()
+        .extend(Layout::from_size_align(0, align_of::<*const u8>() * 2).unwrap())
+        .unwrap();
+
+    let buf = (buf as *mut u8).offset(-(layout_offset as isize));
+    let layout = *(buf as *mut Layout);
+
+    (buf, layout)
+}
+
+/// Calculate a layout for a given size with space for storing a layout at the start.
+/// Returns the layout and the offset to the data.
+fn layout_for_size_prepended(size: usize) -> (Layout, usize) {
+    Layout::new::<Layout>()
+        .extend(Layout::from_size_align(size, align_of::<*const u8>() * 2).unwrap())
+        .unwrap()
+}
+
+/// Store a layout in the pointer, returning a pointer to where the data should be stored.
+unsafe fn store_layout(buf: *mut u8, layout: Layout, offset_to_data: usize) -> *mut c_void {
+    *(buf as *mut Layout) = layout;
+    (buf as *mut u8).offset(offset_to_data as isize) as *mut c_void
+}
+
+/* -------------------------------- string.h -------------------------------- */
+
+#[no_mangle]
+pub unsafe extern "C" fn memcpy(dest: *mut c_void, src: *const c_void, size: usize) -> *mut c_void {
+    std::ptr::copy_nonoverlapping(src, dest, size);
+    dest
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn memmove(
+    dest: *mut c_void,
+    src: *const c_void,
+    size: usize,
+) -> *mut c_void {
+    std::ptr::copy(src, dest, size);
+    dest
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn memset(s: *mut c_void, c: i32, n: usize) -> *mut c_void {
+    let slice = std::slice::from_raw_parts_mut(s as *mut u8, n);
+    slice.fill(c as u8);
+    s
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn memcmp(ptr1: *const c_void, ptr2: *const c_void, n: usize) -> c_int {
+    let s1 = std::slice::from_raw_parts(ptr1 as *const u8, n);
+    let s2 = std::slice::from_raw_parts(ptr2 as *const u8, n);
+
+    for (a, b) in s1.iter().zip(s2.iter()) {
+        if *a != *b {
+            return (*a as i32) - (*b as i32);
+        }
+    }
+
+    0
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn strncmp(ptr1: *const c_void, ptr2: *const c_void, n: usize) -> c_int {
+    let s1 = std::slice::from_raw_parts(ptr1 as *const u8, n);
+    let s2 = std::slice::from_raw_parts(ptr2 as *const u8, n);
+
+    for (a, b) in s1.iter().zip(s2.iter()) {
+        if *a != *b || *a == 0 {
+            return (*a as i32) - (*b as i32);
+        }
+    }
+
+    0
+}
+
+/* -------------------------------- wctype.h -------------------------------- */
+
+#[no_mangle]
+pub unsafe extern "C" fn iswspace(c: c_int) -> bool {
+    char::from_u32(c as u32).map_or(false, |c| c.is_whitespace())
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn iswalnum(c: c_int) -> bool {
+    char::from_u32(c as u32).map_or(false, |c| c.is_alphanumeric())
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn iswdigit(c: c_int) -> bool {
+    char::from_u32(c as u32).map_or(false, |c| c.is_digit(10))
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn iswalpha(c: c_int) -> bool {
+    char::from_u32(c as u32).map_or(false, |c| c.is_alphabetic())
+}
+
+// Note: Not provided by https://github.com/cacticouncil/lilypad, but we needed
+// this one too. We could contribute this back upstream? Note that
+// `towlower()`'s C function docs say it is only guaranteed to work in 1:1
+// mapping cases, so that is what we reimplement here as well.
+// https://en.cppreference.com/w/c/string/wide/towlower
+#[no_mangle]
+pub unsafe extern "C" fn towlower(c: c_int) -> c_int {
+    char::from_u32(c as u32).map_or(0, |c| {
+        c.to_lowercase().next().map(|c| c as i32).unwrap_or(0)
+    })
+}
+
+/* --------------------------------- time.h --------------------------------- */
+
+#[no_mangle]
+pub unsafe extern "C" fn clock() -> u64 {
+    panic!("clock is not supported");
+}
+
+/* --------------------------------- ctype.h -------------------------------- */
+
+#[no_mangle]
+pub unsafe extern "C" fn isprint(c: c_int) -> bool {
+    c >= 32 && c <= 126
+}
+
+/* --------------------------------- stdio.h -------------------------------- */
+
+#[no_mangle]
+pub unsafe extern "C" fn fprintf(_file: *mut c_void, _format: *const c_void, _args: ...) -> c_int {
+    panic!("fprintf is not supported");
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn fputs(_s: *const c_void, _file: *mut c_void) -> c_int {
+    panic!("fputs is not supported");
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn fputc(_c: c_int, _file: *mut c_void) -> c_int {
+    panic!("fputc is not supported");
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn fdopen(_fd: c_int, _mode: *const c_void) -> *mut c_void {
+    panic!("fdopen is not supported");
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn fclose(_file: *mut c_void) -> c_int {
+    panic!("fclose is not supported");
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn fwrite(
+    _ptr: *const c_void,
+    _size: usize,
+    _nmemb: usize,
+    _stream: *mut c_void,
+) -> usize {
+    panic!("fwrite is not supported");
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn vsnprintf(
+    _buf: *mut c_char,
+    _size: usize,
+    _format: *const c_char,
+    _args: ...
+) -> c_int {
+    panic!("vsnprintf is not supported");
+}

crates/wasm-qmd-parser/src/lib.rs

@@ -1,3 +1,15 @@
+// For `vsnprintf()` and `fprintf()`, which are variadic.
+// Otherwise rustc yells at us that we need to enable this.
+#![feature(c_variadic)]
+
+// Provide rust implementation of blessed stdlib functions to
+// tree-sitter itself and any grammars that have `scanner.c`.
+// Here is the list blessed for `scanner.c` usage:
+// https://github.com/tree-sitter/tree-sitter/blob/master/lib/src/wasm/stdlib-symbols.txt
+// But note that we need a few extra for tree-sitter itself.
+#[cfg(target_arch = "wasm32")]
+pub mod c_shim;
+
 mod utils;
 
 use std::io;

crates/wasm-qmd-parser/wasm-sysroot/assert.h

@@ -0,0 +1,4 @@
+#pragma once
+
+#define assert(ignore) ((void)0)
+#define static_assert(cnd, msg) assert(cnd && msg)

crates/wasm-qmd-parser/wasm-sysroot/ctype.h

@@ -0,0 +1,3 @@
+#pragma once
+
+int isprint(int c);

crates/wasm-qmd-parser/wasm-sysroot/endian.h

@@ -0,0 +1,14 @@
+#pragma once
+#include <stdint.h>
+
+#define TS_LITTLE_ENDIAN 1
+
+static inline uint16_t le16toh(uint16_t x) { return x; }
+static inline uint16_t be16toh(uint16_t x)
+{
+#if defined(__GNUC__) || defined(__clang__)
+    return __builtin_bswap16(x);
+#else
+    return (x << 8) | (x >> 8);
+#endif
+}

crates/wasm-qmd-parser/wasm-sysroot/inttypes.h

@@ -0,0 +1,3 @@
+#pragma once
+
+#define PRId32 "d"

crates/wasm-qmd-parser/wasm-sysroot/stdbool.h

@@ -0,0 +1,5 @@
+#pragma once
+
+#define bool _Bool
+#define true 1
+#define false 0