Support table64 extension in classic-interp and AOT running modes (#3811)

Add table64 extension(in Memory64 proposal) support in classic-interp
and AOT running modes, currently still use uint32 to represent table's
initial and maximum size to keep AOT ABI unchanged.

Author: TianlongLiang

core/iwasm/aot/aot_loader.c

@@ -1382,6 +1382,12 @@ load_table_list(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module,
     for (i = 0; i < module->table_count; i++, table++) {
         read_uint8(buf, buf_end, table->table_type.elem_type);
         read_uint8(buf, buf_end, table->table_type.flags);
+
+        if (!wasm_table_check_flags(table->table_type.flags, error_buf,
+                                    error_buf_size, true)) {
+            return false;
+        }
+
         read_uint8(buf, buf_end, table->table_type.possible_grow);
 #if WASM_ENABLE_GC != 0
         if (wasm_is_type_multi_byte_type(table->table_type.elem_type)) {

core/iwasm/aot/aot_runtime.c

@@ -737,18 +737,24 @@ tables_instantiate(AOTModuleInstance *module_inst, AOTModule *module,
 
 #if WASM_ENABLE_REF_TYPES != 0
         bh_assert(
-            table_seg->offset.init_expr_type == INIT_EXPR_TYPE_I32_CONST
+            table_seg->offset.init_expr_type
+                == (tbl_inst->is_table64 ? INIT_EXPR_TYPE_I64_CONST
+                                         : INIT_EXPR_TYPE_I32_CONST)
             || table_seg->offset.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL
             || table_seg->offset.init_expr_type == INIT_EXPR_TYPE_FUNCREF_CONST
             || table_seg->offset.init_expr_type
                    == INIT_EXPR_TYPE_REFNULL_CONST);
 #else
-        bh_assert(table_seg->offset.init_expr_type == INIT_EXPR_TYPE_I32_CONST
+        bh_assert(table_seg->offset.init_expr_type
+                      == (tbl_inst->is_table64 ? INIT_EXPR_TYPE_I64_CONST
+                                               : INIT_EXPR_TYPE_I32_CONST)
                   || table_seg->offset.init_expr_type
                          == INIT_EXPR_TYPE_GET_GLOBAL);
 #endif
 
         /* Resolve table data base offset */
+        /* TODO: The table64 current implementation assumes table max size
+         * UINT32_MAX, so the offset conversion here is safe */
         if (table_seg->offset.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) {
             global_index = table_seg->offset.u.global_index;
 

core/iwasm/common/wasm_loader_common.c

@@ -19,6 +19,37 @@ wasm_loader_set_error_buf(char *error_buf, uint32 error_buf_size,
     }
 }
 
+#if WASM_ENABLE_MEMORY64 != 0
+bool
+check_memory64_flags_consistency(WASMModule *module, char *error_buf,
+                                 uint32 error_buf_size, bool is_aot)
+{
+    uint32 i;
+    bool wasm64_flag, all_wasm64 = true, none_wasm64 = true;
+
+    for (i = 0; i < module->import_memory_count; ++i) {
+        wasm64_flag =
+            module->import_memories[i].u.memory.mem_type.flags & MEMORY64_FLAG;
+        all_wasm64 &= wasm64_flag;
+        none_wasm64 &= !wasm64_flag;
+    }
+
+    for (i = 0; i < module->memory_count; ++i) {
+        wasm64_flag = module->memories[i].flags & MEMORY64_FLAG;
+        all_wasm64 &= wasm64_flag;
+        none_wasm64 &= !wasm64_flag;
+    }
+
+    if (!(all_wasm64 || none_wasm64)) {
+        wasm_loader_set_error_buf(
+            error_buf, error_buf_size,
+            "inconsistent limits wasm64 flags for memory sections", is_aot);
+        return false;
+    }
+    return true;
+}
+#endif
+
 bool
 wasm_memory_check_flags(const uint8 mem_flag, char *error_buf,
                         uint32 error_buf_size, bool is_aot)
@@ -60,6 +91,37 @@ wasm_memory_check_flags(const uint8 mem_flag, char *error_buf,
     return true;
 }
 
+bool
+wasm_table_check_flags(const uint8 table_flag, char *error_buf,
+                       uint32 error_buf_size, bool is_aot)
+{
+    /* Check whether certain features indicated by mem_flag are enabled in
+     * runtime */
+    if (table_flag > MAX_TABLE_SIZE_FLAG) {
+        if (table_flag & SHARED_TABLE_FLAG) {
+            wasm_loader_set_error_buf(error_buf, error_buf_size,
+                                      "tables cannot be shared", is_aot);
+        }
+#if WASM_ENABLE_MEMORY64 == 0
+        if (table_flag & TABLE64_FLAG) {
+            wasm_loader_set_error_buf(error_buf, error_buf_size,
+                                      "invalid limits flags(table64 flag was "
+                                      "found, please enable memory64)",
+                                      is_aot);
+            return false;
+        }
+#endif
+    }
+
+    if (table_flag > MAX_TABLE_SIZE_FLAG + TABLE64_FLAG) {
+        wasm_loader_set_error_buf(error_buf, error_buf_size,
+                                  "invalid limits flags", is_aot);
+        return false;
+    }
+
+    return true;
+}
+
 /*
  * compare with a bigger type set in `wasm_value_type_size_internal()`,
  * this function will only cover global value type, function's param

core/iwasm/common/wasm_loader_common.h

@@ -13,10 +13,22 @@
 extern "C" {
 #endif
 
+#if WASM_ENABLE_MEMORY64 != 0
+/* check consistency of memory64 flags across all memories,
+ * they must be either all wasm64 or all wasm32 */
+bool
+check_memory64_flags_consistency(WASMModule *module, char *error_buf,
+                                 uint32 error_buf_size, bool is_aot);
+#endif
+
 bool
 wasm_memory_check_flags(const uint8 mem_flag, char *error_buf,
                         uint32 error_buf_size, bool is_aot);
 
+bool
+wasm_table_check_flags(const uint8 table_flag, char *error_buf,
+                       uint32 error_buf_size, bool is_aot);
+
 bool
 is_valid_value_type(uint8 value_tpye);
 

core/iwasm/compilation/aot_compiler.c

@@ -146,9 +146,20 @@ aot_validate_wasm(AOTCompContext *comp_ctx)
     }
 
 #if WASM_ENABLE_MEMORY64 != 0
-    if (comp_ctx->pointer_size < sizeof(uint64) && IS_MEMORY64) {
-        aot_set_last_error("Compiling wasm64 to 32bit platform is not allowed");
-        return false;
+    if (comp_ctx->pointer_size < sizeof(uint64)) {
+        if (IS_MEMORY64) {
+            aot_set_last_error("Compiling wasm64(contains i64 memory section) "
+                               "to 32bit platform is not allowed");
+            return false;
+        }
+
+        for (uint32 i = 0; i < comp_ctx->comp_data->table_count; ++i) {
+            if (IS_TABLE64(i)) {
+                aot_set_last_error("Compiling wasm64(contains i64 table "
+                                   "section) to 32bit platform is not allowed");
+                return false;
+            }
+        }
     }
 #endif
 

core/iwasm/compilation/aot_compiler.h

@@ -532,8 +532,13 @@ set_local_gc_ref(AOTCompFrame *frame, int n, LLVMValueRef value, uint8 ref_type)
 #define IS_MEMORY64 (comp_ctx->comp_data->memories[0].flags & MEMORY64_FLAG)
 #define MEMORY64_COND_VALUE(VAL_IF_ENABLED, VAL_IF_DISABLED) \
     (IS_MEMORY64 ? VAL_IF_ENABLED : VAL_IF_DISABLED)
+#define IS_TABLE64(i) \
+    (comp_ctx->comp_data->tables[i].table_type.flags & TABLE64_FLAG)
+#define TABLE64_COND_VALUE(i, VAL_IF_ENABLED, VAL_IF_DISABLED) \
+    (IS_TABLE64(i) ? VAL_IF_ENABLED : VAL_IF_DISABLED)
 #else
 #define MEMORY64_COND_VALUE(VAL_IF_ENABLED, VAL_IF_DISABLED) (VAL_IF_DISABLED)
+#define TABLE64_COND_VALUE(i, VAL_IF_ENABLED, VAL_IF_DISABLED) (VAL_IF_DISABLED)
 #endif
 
 #define POP_I32(v) POP(v, VALUE_TYPE_I32)
@@ -548,6 +553,9 @@ set_local_gc_ref(AOTCompFrame *frame, int n, LLVMValueRef value, uint8 ref_type)
     POP(v, MEMORY64_COND_VALUE(VALUE_TYPE_I64, VALUE_TYPE_I32))
 #define POP_PAGE_COUNT(v) \
     POP(v, MEMORY64_COND_VALUE(VALUE_TYPE_I64, VALUE_TYPE_I32))
+#define POP_TBL_ELEM_IDX(v) \
+    POP(v, TABLE64_COND_VALUE(tbl_idx, VALUE_TYPE_I64, VALUE_TYPE_I32))
+#define POP_TBL_ELEM_LEN(v) POP_TBL_ELEM_IDX(v)
 
 #define POP_COND(llvm_value)                                                   \
     do {                                                                       \
@@ -613,6 +621,9 @@ set_local_gc_ref(AOTCompFrame *frame, int n, LLVMValueRef value, uint8 ref_type)
 #define PUSH_GC_REF(v) PUSH(v, VALUE_TYPE_GC_REF)
 #define PUSH_PAGE_COUNT(v) \
     PUSH(v, MEMORY64_COND_VALUE(VALUE_TYPE_I64, VALUE_TYPE_I32))
+#define PUSH_TBL_ELEM_IDX(v) \
+    PUSH(v, TABLE64_COND_VALUE(tbl_idx, VALUE_TYPE_I64, VALUE_TYPE_I32))
+#define PUSH_TBL_ELEM_LEN(v) PUSH_TBL_ELEM_IDX(v)
 
 #define SET_CONST(v)                                                          \
     do {                                                                      \

core/iwasm/compilation/aot_emit_function.c

@@ -2089,6 +2089,9 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
     LLVMValueRef ext_ret_offset, ext_ret_ptr, ext_ret, res;
     LLVMValueRef *param_values = NULL, *value_rets = NULL;
     LLVMValueRef *result_phis = NULL, value_ret, import_func_count;
+#if WASM_ENABLE_MEMORY64 != 0
+    LLVMValueRef u32_max, u32_cmp_result;
+#endif
     LLVMTypeRef *param_types = NULL, ret_type;
     LLVMTypeRef llvm_func_type, llvm_func_ptr_type;
     LLVMTypeRef ext_ret_ptr_type;
@@ -2153,7 +2156,7 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
     func_param_count = func_type->param_count;
     func_result_count = func_type->result_count;
 
-    POP_I32(elem_idx);
+    POP_TBL_ELEM_IDX(elem_idx);
 
     /* get the cur size of the table instance */
     if (!(offset = I32_CONST(get_tbl_inst_offset(comp_ctx, func_ctx, tbl_idx)
@@ -2182,14 +2185,47 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
         goto fail;
     }
 
+#if WASM_ENABLE_MEMORY64 != 0
+    /* Check if elem index >= UINT32_MAX */
+    if (IS_TABLE64(tbl_idx)) {
+        if (!(u32_max = I64_CONST(UINT32_MAX))) {
+            aot_set_last_error("llvm build const failed");
+            goto fail;
+        }
+        if (!(u32_cmp_result =
+                  LLVMBuildICmp(comp_ctx->builder, LLVMIntUGE, elem_idx,
+                                u32_max, "cmp_elem_idx_u32_max"))) {
+            aot_set_last_error("llvm build icmp failed.");
+            goto fail;
+        }
+        if (!(elem_idx = LLVMBuildTrunc(comp_ctx->builder, elem_idx, I32_TYPE,
+                                        "elem_idx_i32"))) {
+            aot_set_last_error("llvm build trunc failed.");
+            goto fail;
+        }
+    }
+#endif
+
     /* Check if (uint32)elem index >= table size */
     if (!(cmp_elem_idx = LLVMBuildICmp(comp_ctx->builder, LLVMIntUGE, elem_idx,
                                        table_size_const, "cmp_elem_idx"))) {
         aot_set_last_error("llvm build icmp failed.");
         goto fail;
     }
 
-    /* Throw exception if elem index >= table size */
+#if WASM_ENABLE_MEMORY64 != 0
+    if (IS_TABLE64(tbl_idx)) {
+        if (!(cmp_elem_idx =
+                  LLVMBuildOr(comp_ctx->builder, cmp_elem_idx, u32_cmp_result,
+                              "larger_than_u32_max_or_cur_size"))) {
+            aot_set_last_error("llvm build or failed.");
+            goto fail;
+        }
+    }
+#endif
+
+    /* Throw exception if elem index >= table size or elem index >= UINT32_MAX
+     */
     if (!(check_elem_idx_succ = LLVMAppendBasicBlockInContext(
               comp_ctx->context, func_ctx->func, "check_elem_idx_succ"))) {
         aot_set_last_error("llvm add basic block failed.");