diff --git a/.gitignore b/.gitignore index 8b9e2b7d..b0dc43c5 100644 --- a/.gitignore +++ b/.gitignore @@ -28,4 +28,7 @@ tests/wamr-test-suites/workspace samples/socket-api/wasm-src/inc/pthread.h -**/__pycache__ \ No newline at end of file +**/__pycache__ + +# ignore benchmarks generated +tests/benchmarks/coremark/coremark* diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index c99d1564..289242b2 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -3,7 +3,6 @@ string(TOUPPER ${WAMR_BUILD_TARGET} WAMR_BUILD_TARGET) - # Add definitions for the build target if (WAMR_BUILD_TARGET STREQUAL "X86_64") add_definitions(-DBUILD_TARGET_X86_64) @@ -85,33 +84,29 @@ endif () endif () if (WAMR_BUILD_JIT EQUAL 1) - if (WAMR_BUILD_AOT EQUAL 1) - add_definitions("-DWASM_ENABLE_JIT=1") - if (NOT WAMR_BUILD_LAZY_JIT EQUAL 0) - # Enable Lazy JIT by default - set (WAMR_BUILD_LAZY_JIT 1) - add_definitions("-DWASM_ENABLE_LAZY_JIT=1") + add_definitions("-DWASM_ENABLE_JIT=1") + if (NOT WAMR_BUILD_LAZY_JIT EQUAL 0) + # Enable Lazy JIT by default + set (WAMR_BUILD_LAZY_JIT 1) + add_definitions("-DWASM_ENABLE_LAZY_JIT=1") + endif () + if (NOT DEFINED LLVM_DIR) + set (LLVM_SRC_ROOT "${WAMR_ROOT_DIR}/core/deps/llvm") + set (LLVM_BUILD_ROOT "${LLVM_SRC_ROOT}/build") + if (WAMR_BUILD_PLATFORM STREQUAL "windows") + set (LLVM_BUILD_ROOT "${LLVM_SRC_ROOT}/win32build") endif () - if (NOT DEFINED LLVM_DIR) - set (LLVM_SRC_ROOT "${WAMR_ROOT_DIR}/core/deps/llvm") - set (LLVM_BUILD_ROOT "${LLVM_SRC_ROOT}/build") - if (WAMR_BUILD_PLATFORM STREQUAL "windows") - set (LLVM_BUILD_ROOT "${LLVM_SRC_ROOT}/win32build") - endif () - if (NOT EXISTS "${LLVM_BUILD_ROOT}") + if (NOT EXISTS "${LLVM_BUILD_ROOT}") message (FATAL_ERROR "Cannot find LLVM dir: ${LLVM_BUILD_ROOT}") - endif () - set (CMAKE_PREFIX_PATH "${LLVM_BUILD_ROOT};${CMAKE_PREFIX_PATH}") endif () - find_package(LLVM REQUIRED CONFIG) - include_directories(${LLVM_INCLUDE_DIRS}) - add_definitions(${LLVM_DEFINITIONS}) - message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") - message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") - else () - set (WAMR_BUILD_JIT 0) - message ("-- WAMR JIT disabled due to WAMR AOT is disabled") - endif () + set (CMAKE_PREFIX_PATH "${LLVM_BUILD_ROOT};${CMAKE_PREFIX_PATH}") + set (LLVM_DIR ${LLVM_BUILD_ROOT}/lib/cmake/llvm) + endif () + find_package(LLVM REQUIRED CONFIG) + include_directories(${LLVM_INCLUDE_DIRS}) + add_definitions(${LLVM_DEFINITIONS}) + message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") + message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") else () unset (LLVM_AVAILABLE_LIBS) endif () @@ -139,9 +134,6 @@ message (" Build as target ${WAMR_BUILD_TARGET}") message (" CMAKE_BUILD_TYPE " ${CMAKE_BUILD_TYPE}) if (WAMR_BUILD_INTERP EQUAL 1) message (" WAMR Interpreter enabled") -elseif (WAMR_BUILD_JIT EQUAL 1) - set (WAMR_BUILD_INTERP 1) - message (" WAMR Interpreter enabled due to WAMR JIT is enabled") else () message (" WAMR Interpreter disabled") endif () @@ -150,16 +142,19 @@ if (WAMR_BUILD_AOT EQUAL 1) else () message (" WAMR AOT disabled") endif () +if (WAMR_BUILD_FAST_JIT EQUAL 1) + message (" WAMR Fast JIT enabled") +else () + message (" WAMR Fast JIT disabled") +endif () if (WAMR_BUILD_JIT EQUAL 1) if (WAMR_BUILD_LAZY_JIT EQUAL 1) message (" WAMR LLVM Orc Lazy JIT enabled") else () message (" WAMR LLVM MC JIT enabled") endif () -elseif (WAMR_BUILD_FAST_JIT EQUAL 1) - message (" WAMR Fast JIT enabled") else () - message (" WAMR JIT disabled") + message (" WAMR LLVM JIT disabled") endif () if (WAMR_BUILD_LIBC_BUILTIN EQUAL 1) message (" Libc builtin enabled") diff --git a/build-scripts/runtime_lib.cmake b/build-scripts/runtime_lib.cmake index 2a46d432..c5195895 100644 --- a/build-scripts/runtime_lib.cmake +++ b/build-scripts/runtime_lib.cmake @@ -1,7 +1,6 @@ # Copyright (C) 2019 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - if (NOT DEFINED WAMR_ROOT_DIR) set (WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../) endif () @@ -50,23 +49,28 @@ if (NOT DEFINED WAMR_BUILD_TARGET) endif () ################ optional according to settings ################ -if (WAMR_BUILD_INTERP EQUAL 1 OR WAMR_BUILD_JIT EQUAL 1 - OR WAMR_BUILD_FAST_JIT EQUAL 1) - if (WAMR_BUILD_FAST_JIT EQUAL 1 OR WAMR_BUILD_JIT EQUAL 1) - set (WAMR_BUILD_FAST_INTERP 0) - endif () +if (WAMR_BUILD_FAST_JIT EQUAL 1 OR WAMR_BUILD_JIT EQUAL 1) + # Enable classic interpreter if Fast JIT or LLVM JIT is enabled + set (WAMR_BUILD_INTERP 1) + set (WAMR_BUILD_FAST_INTERP 0) +endif () + +if (WAMR_BUILD_INTERP EQUAL 1) include (${IWASM_DIR}/interpreter/iwasm_interp.cmake) endif () +if (WAMR_BUILD_FAST_JIT EQUAL 1) + include (${IWASM_DIR}/fast-jit/iwasm_fast_jit.cmake) +endif () + +if (WAMR_BUILD_JIT EQUAL 1) + # Enable AOT if LLVM JIT is enabled + set (WAMR_BUILD_AOT 1) + include (${IWASM_DIR}/compilation/iwasm_compl.cmake) +endif () + if (WAMR_BUILD_AOT EQUAL 1) include (${IWASM_DIR}/aot/iwasm_aot.cmake) - if (WAMR_BUILD_JIT EQUAL 1) - include (${IWASM_DIR}/compilation/iwasm_compl.cmake) - endif () -endif () - -if (NOT WAMR_BUILD_JIT EQUAL 1 AND WAMR_BUILD_FAST_JIT EQUAL 1) - include (${IWASM_DIR}/fast-jit/iwasm_fast_jit.cmake) endif () if (WAMR_BUILD_APP_FRAMEWORK EQUAL 1) diff --git a/core/iwasm/aot/aot_intrinsic.h b/core/iwasm/aot/aot_intrinsic.h index 92dccea0..c6765ee0 100644 --- a/core/iwasm/aot/aot_intrinsic.h +++ b/core/iwasm/aot/aot_intrinsic.h @@ -7,7 +7,7 @@ #define _AOT_INTRINSIC_H #include "aot_runtime.h" -#if WASM_ENABLE_WAMR_COMPILER != 0 +#if WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0 #include "aot_llvm.h" #endif diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index fca926ab..dca35685 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -10,10 +10,6 @@ #include "../common/wasm_runtime_common.h" #include "../common/wasm_native.h" #include "../compilation/aot.h" -#if WASM_ENABLE_JIT != 0 -#include "../compilation/aot_llvm.h" -#include "../interpreter/wasm_loader.h" -#endif #if WASM_ENABLE_DEBUG_AOT != 0 #include "debug/elf_parser.h" @@ -716,23 +712,19 @@ fail: } static void -destroy_import_memories(AOTImportMemory *import_memories, bool is_jit_mode) +destroy_import_memories(AOTImportMemory *import_memories) { - if (!is_jit_mode) - wasm_runtime_free(import_memories); + wasm_runtime_free(import_memories); } static void -destroy_mem_init_data_list(AOTMemInitData **data_list, uint32 count, - bool is_jit_mode) +destroy_mem_init_data_list(AOTMemInitData **data_list, uint32 count) { - if (!is_jit_mode) { - uint32 i; - for (i = 0; i < count; i++) - if (data_list[i]) - wasm_runtime_free(data_list[i]); - wasm_runtime_free(data_list); - } + uint32 i; + for (i = 0; i < count; i++) + if (data_list[i]) + wasm_runtime_free(data_list[i]); + wasm_runtime_free(data_list); } static bool @@ -828,30 +820,25 @@ fail: } static void -destroy_import_tables(AOTImportTable *import_tables, bool is_jit_mode) +destroy_import_tables(AOTImportTable *import_tables) { - if (!is_jit_mode) - wasm_runtime_free(import_tables); + wasm_runtime_free(import_tables); } static void -destroy_tables(AOTTable *tables, bool is_jit_mode) +destroy_tables(AOTTable *tables) { - if (!is_jit_mode) - wasm_runtime_free(tables); + wasm_runtime_free(tables); } static void -destroy_table_init_data_list(AOTTableInitData **data_list, uint32 count, - bool is_jit_mode) +destroy_table_init_data_list(AOTTableInitData **data_list, uint32 count) { - if (!is_jit_mode) { - uint32 i; - for (i = 0; i < count; i++) - if (data_list[i]) - wasm_runtime_free(data_list[i]); - wasm_runtime_free(data_list); - } + uint32 i; + for (i = 0; i < count; i++) + if (data_list[i]) + wasm_runtime_free(data_list[i]); + wasm_runtime_free(data_list); } static bool @@ -1003,15 +990,13 @@ fail: } static void -destroy_func_types(AOTFuncType **func_types, uint32 count, bool is_jit_mode) +destroy_func_types(AOTFuncType **func_types, uint32 count) { - if (!is_jit_mode) { - uint32 i; - for (i = 0; i < count; i++) - if (func_types[i]) - wasm_runtime_free(func_types[i]); - wasm_runtime_free(func_types); - } + uint32 i; + for (i = 0; i < count; i++) + if (func_types[i]) + wasm_runtime_free(func_types[i]); + wasm_runtime_free(func_types); } static bool @@ -1094,10 +1079,9 @@ fail: } static void -destroy_import_globals(AOTImportGlobal *import_globals, bool is_jit_mode) +destroy_import_globals(AOTImportGlobal *import_globals) { - if (!is_jit_mode) - wasm_runtime_free(import_globals); + wasm_runtime_free(import_globals); } static bool @@ -1177,10 +1161,9 @@ fail: } static void -destroy_globals(AOTGlobal *globals, bool is_jit_mode) +destroy_globals(AOTGlobal *globals) { - if (!is_jit_mode) - wasm_runtime_free(globals); + wasm_runtime_free(globals); } static bool @@ -1259,10 +1242,9 @@ fail: } static void -destroy_import_funcs(AOTImportFunc *import_funcs, bool is_jit_mode) +destroy_import_funcs(AOTImportFunc *import_funcs) { - if (!is_jit_mode) - wasm_runtime_free(import_funcs); + wasm_runtime_free(import_funcs); } static bool @@ -1652,10 +1634,9 @@ fail: } static void -destroy_exports(AOTExport *exports, bool is_jit_mode) +destroy_exports(AOTExport *exports) { - if (!is_jit_mode) - wasm_runtime_free(exports); + wasm_runtime_free(exports); } static bool @@ -2826,427 +2807,46 @@ aot_load_from_aot_file(const uint8 *buf, uint32 size, char *error_buf, return module; } -#if WASM_ENABLE_JIT != 0 -#if WASM_ENABLE_LAZY_JIT != 0 -/* Orc JIT thread arguments */ -typedef struct OrcJitThreadArg { - AOTCompData *comp_data; - AOTCompContext *comp_ctx; - AOTModule *module; - int32 group_idx; - int32 group_stride; -} OrcJitThreadArg; - -static bool orcjit_stop_compiling = false; -static korp_tid orcjit_threads[WASM_LAZY_JIT_COMPILE_THREAD_NUM]; -static OrcJitThreadArg orcjit_thread_args[WASM_LAZY_JIT_COMPILE_THREAD_NUM]; - -static void * -orcjit_thread_callback(void *arg) -{ - LLVMErrorRef error; - LLVMOrcJITTargetAddress func_addr = 0; - OrcJitThreadArg *thread_arg = (OrcJitThreadArg *)arg; - AOTCompData *comp_data = thread_arg->comp_data; - AOTCompContext *comp_ctx = thread_arg->comp_ctx; - AOTModule *module = thread_arg->module; - char func_name[32]; - int32 i; - - /* Compile wasm functions of this group */ - for (i = thread_arg->group_idx; i < (int32)comp_data->func_count; - i += thread_arg->group_stride) { - if (!module->func_ptrs[i]) { - snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i); - if ((error = LLVMOrcLLJITLookup(comp_ctx->orc_lazyjit, &func_addr, - func_name))) { - char *err_msg = LLVMGetErrorMessage(error); - os_printf("failed to compile orc jit function: %s", err_msg); - LLVMDisposeErrorMessage(err_msg); - break; - } - /** - * No need to lock the func_ptr[func_idx] here as it is basic - * data type, the load/store for it can be finished by one cpu - * instruction, and there can be only one cpu instruction - * loading/storing at the same time. - */ - module->func_ptrs[i] = (void *)func_addr; - } - if (orcjit_stop_compiling) { - break; - } - } - - /* Try to compile functions that haven't been compiled by other threads */ - for (i = (int32)comp_data->func_count - 1; i > 0; i--) { - if (orcjit_stop_compiling) { - break; - } - if (!module->func_ptrs[i]) { - snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i); - if ((error = LLVMOrcLLJITLookup(comp_ctx->orc_lazyjit, &func_addr, - func_name))) { - char *err_msg = LLVMGetErrorMessage(error); - os_printf("failed to compile orc jit function: %s", err_msg); - LLVMDisposeErrorMessage(err_msg); - break; - } - module->func_ptrs[i] = (void *)func_addr; - } - } - - return NULL; -} - -static void -orcjit_stop_compile_threads() -{ - uint32 i; - - orcjit_stop_compiling = true; - for (i = 0; i < WASM_LAZY_JIT_COMPILE_THREAD_NUM; i++) { - os_thread_join(orcjit_threads[i], NULL); - } -} -#endif - -static AOTModule * -aot_load_from_comp_data(AOTCompData *comp_data, AOTCompContext *comp_ctx, - char *error_buf, uint32 error_buf_size) -{ - uint32 i; - uint64 size; - char func_name[32]; - AOTModule *module; - - /* Allocate memory for module */ - if (!(module = - loader_malloc(sizeof(AOTModule), error_buf, error_buf_size))) { - return NULL; - } - - module->module_type = Wasm_Module_AoT; - - module->import_memory_count = comp_data->import_memory_count; - module->import_memories = comp_data->import_memories; - - module->memory_count = comp_data->memory_count; - if (module->memory_count) { - size = sizeof(AOTMemory) * (uint64)module->memory_count; - if (!(module->memories = - loader_malloc(size, error_buf, error_buf_size))) { - goto fail1; - } - - bh_memcpy_s(module->memories, (uint32)size, comp_data->memories, - (uint32)size); - } - - module->mem_init_data_list = comp_data->mem_init_data_list; - module->mem_init_data_count = comp_data->mem_init_data_count; - - module->import_table_count = comp_data->import_table_count; - module->import_tables = comp_data->import_tables; - - module->table_count = comp_data->table_count; - module->tables = comp_data->tables; - - module->table_init_data_list = comp_data->table_init_data_list; - module->table_init_data_count = comp_data->table_init_data_count; - - module->func_type_count = comp_data->func_type_count; - module->func_types = comp_data->func_types; - - module->import_global_count = comp_data->import_global_count; - module->import_globals = comp_data->import_globals; - - module->global_count = comp_data->global_count; - module->globals = comp_data->globals; - - module->global_count = comp_data->global_count; - module->globals = comp_data->globals; - - module->global_data_size = comp_data->global_data_size; - - module->import_func_count = comp_data->import_func_count; - module->import_funcs = comp_data->import_funcs; - - module->func_count = comp_data->func_count; - - /* Allocate memory for function pointers */ - size = (uint64)module->func_count * sizeof(void *); - if (size > 0 - && !(module->func_ptrs = - loader_malloc(size, error_buf, error_buf_size))) { - goto fail2; - } - -#if WASM_ENABLE_LAZY_JIT != 0 - /* Create threads to compile the wasm functions */ - for (i = 0; i < WASM_LAZY_JIT_COMPILE_THREAD_NUM; i++) { - orcjit_thread_args[i].comp_data = comp_data; - orcjit_thread_args[i].comp_ctx = comp_ctx; - orcjit_thread_args[i].module = module; - orcjit_thread_args[i].group_idx = (int32)i; - orcjit_thread_args[i].group_stride = WASM_LAZY_JIT_COMPILE_THREAD_NUM; - if (os_thread_create(&orcjit_threads[i], orcjit_thread_callback, - (void *)&orcjit_thread_args[i], - APP_THREAD_STACK_SIZE_DEFAULT) - != 0) { - uint32 j; - - set_error_buf(error_buf, error_buf_size, - "create orcjit compile thread failed"); - /* Terminate the threads created */ - orcjit_stop_compiling = true; - for (j = 0; j < i; j++) { - os_thread_join(orcjit_threads[j], NULL); - } - goto fail3; - } - } -#else - /* Resolve function addresses */ - bh_assert(comp_ctx->exec_engine); - for (i = 0; i < comp_data->func_count; i++) { - snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i); - if (!(module->func_ptrs[i] = (void *)LLVMGetFunctionAddress( - comp_ctx->exec_engine, func_name))) { - set_error_buf(error_buf, error_buf_size, - "get function address failed"); - goto fail3; - } - } -#endif /* WASM_ENABLE_LAZY_JIT != 0 */ - - /* Allocation memory for function type indexes */ - size = (uint64)module->func_count * sizeof(uint32); - if (size > 0 - && !(module->func_type_indexes = - loader_malloc(size, error_buf, error_buf_size))) { - goto fail4; - } - for (i = 0; i < comp_data->func_count; i++) - module->func_type_indexes[i] = comp_data->funcs[i]->func_type_index; - - module->export_count = comp_data->wasm_module->export_count; - module->exports = comp_data->wasm_module->exports; - - module->start_func_index = comp_data->start_func_index; - if (comp_data->start_func_index != (uint32)-1) { - bh_assert(comp_data->start_func_index - < module->import_func_count + module->func_count); - /* TODO: fix issue that start func cannot be import func */ - if (comp_data->start_func_index >= module->import_func_count) { -#if WASM_ENABLE_LAZY_JIT != 0 - if (!module->func_ptrs[comp_data->start_func_index - - module->import_func_count]) { - LLVMErrorRef error; - LLVMOrcJITTargetAddress func_addr = 0; - - snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, - comp_data->start_func_index - - module->import_func_count); - if ((error = LLVMOrcLLJITLookup(comp_ctx->orc_lazyjit, - &func_addr, func_name))) { - char *err_msg = LLVMGetErrorMessage(error); - set_error_buf_v(error_buf, error_buf_size, - "failed to compile orc jit function: %s", - err_msg); - LLVMDisposeErrorMessage(err_msg); - goto fail5; - } - module->func_ptrs[comp_data->start_func_index - - module->import_func_count] = - (void *)func_addr; - } -#endif - module->start_function = - module->func_ptrs[comp_data->start_func_index - - module->import_func_count]; - } - } - - module->malloc_func_index = comp_data->malloc_func_index; - module->free_func_index = comp_data->free_func_index; - module->retain_func_index = comp_data->retain_func_index; - - module->aux_data_end_global_index = comp_data->aux_data_end_global_index; - module->aux_data_end = comp_data->aux_data_end; - module->aux_heap_base_global_index = comp_data->aux_heap_base_global_index; - module->aux_heap_base = comp_data->aux_heap_base; - module->aux_stack_top_global_index = comp_data->aux_stack_top_global_index; - module->aux_stack_bottom = comp_data->aux_stack_bottom; - module->aux_stack_size = comp_data->aux_stack_size; - - module->code = NULL; - module->code_size = 0; - - module->is_jit_mode = true; - - module->wasm_module = comp_data->wasm_module; - module->comp_ctx = comp_ctx; - module->comp_data = comp_data; - -#if WASM_ENABLE_LIBC_WASI != 0 - module->import_wasi_api = comp_data->wasm_module->import_wasi_api; -#endif - - return module; - -#if WASM_ENABLE_LAZY_JIT != 0 -fail5: - if (module->func_type_indexes) - wasm_runtime_free(module->func_type_indexes); -#endif - -fail4: -#if WASM_ENABLE_LAZY_JIT != 0 - /* Terminate all threads before free module->func_ptrs */ - orcjit_stop_compile_threads(); -#endif -fail3: - if (module->func_ptrs) - wasm_runtime_free(module->func_ptrs); -fail2: - if (module->memory_count > 0) - wasm_runtime_free(module->memories); -fail1: - wasm_runtime_free(module); - return NULL; -} - -AOTModule * -aot_convert_wasm_module(WASMModule *wasm_module, char *error_buf, - uint32 error_buf_size) -{ - AOTCompData *comp_data; - AOTCompContext *comp_ctx; - AOTModule *aot_module; - AOTCompOption option = { 0 }; - char *aot_last_error; - - comp_data = aot_create_comp_data(wasm_module); - if (!comp_data) { - aot_last_error = aot_get_last_error(); - bh_assert(aot_last_error != NULL); - set_error_buf(error_buf, error_buf_size, aot_last_error); - return NULL; - } - - option.is_jit_mode = true; - option.opt_level = 3; - option.size_level = 3; -#if WASM_ENABLE_BULK_MEMORY != 0 - option.enable_bulk_memory = true; -#endif -#if WASM_ENABLE_THREAD_MGR != 0 - option.enable_thread_mgr = true; -#endif -#if WASM_ENABLE_TAIL_CALL != 0 - option.enable_tail_call = true; -#endif -#if WASM_ENABLE_SIMD != 0 - option.enable_simd = true; -#endif -#if WASM_ENABLE_REF_TYPES != 0 - option.enable_ref_types = true; -#endif - option.enable_aux_stack_check = true; -#if (WASM_ENABLE_PERF_PROFILING != 0) || (WASM_ENABLE_DUMP_CALL_STACK != 0) - option.enable_aux_stack_frame = true; -#endif - - comp_ctx = aot_create_comp_context(comp_data, &option); - if (!comp_ctx) { - aot_last_error = aot_get_last_error(); - bh_assert(aot_last_error != NULL); - set_error_buf(error_buf, error_buf_size, aot_last_error); - goto fail1; - } - - if (!aot_compile_wasm(comp_ctx)) { - aot_last_error = aot_get_last_error(); - bh_assert(aot_last_error != NULL); - set_error_buf(error_buf, error_buf_size, aot_last_error); - goto fail2; - } - - aot_module = - aot_load_from_comp_data(comp_data, comp_ctx, error_buf, error_buf_size); - if (!aot_module) { - goto fail2; - } - - return aot_module; - -fail2: - aot_destroy_comp_context(comp_ctx); -fail1: - aot_destroy_comp_data(comp_data); - return NULL; -} -#endif - void aot_unload(AOTModule *module) { -#if WASM_ENABLE_JIT != 0 -#if WASM_ENABLE_LAZY_JIT != 0 - orcjit_stop_compile_threads(); -#endif - - if (module->comp_data) - aot_destroy_comp_data(module->comp_data); - - if (module->comp_ctx) - aot_destroy_comp_context(module->comp_ctx); - - if (module->wasm_module) - wasm_loader_unload(module->wasm_module); -#endif - if (module->import_memories) - destroy_import_memories(module->import_memories, module->is_jit_mode); + destroy_import_memories(module->import_memories); if (module->memories) wasm_runtime_free(module->memories); if (module->mem_init_data_list) destroy_mem_init_data_list(module->mem_init_data_list, - module->mem_init_data_count, - module->is_jit_mode); + module->mem_init_data_count); if (module->native_symbol_list) wasm_runtime_free(module->native_symbol_list); if (module->import_tables) - destroy_import_tables(module->import_tables, module->is_jit_mode); + destroy_import_tables(module->import_tables); if (module->tables) - destroy_tables(module->tables, module->is_jit_mode); + destroy_tables(module->tables); if (module->table_init_data_list) destroy_table_init_data_list(module->table_init_data_list, - module->table_init_data_count, - module->is_jit_mode); + module->table_init_data_count); if (module->func_types) - destroy_func_types(module->func_types, module->func_type_count, - module->is_jit_mode); + destroy_func_types(module->func_types, module->func_type_count); if (module->import_globals) - destroy_import_globals(module->import_globals, module->is_jit_mode); + destroy_import_globals(module->import_globals); if (module->globals) - destroy_globals(module->globals, module->is_jit_mode); + destroy_globals(module->globals); if (module->import_funcs) - destroy_import_funcs(module->import_funcs, module->is_jit_mode); + destroy_import_funcs(module->import_funcs); if (module->exports) - destroy_exports(module->exports, module->is_jit_mode); + destroy_exports(module->exports); if (module->func_type_indexes) wasm_runtime_free(module->func_type_indexes); diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index c1ae5e41..13bdc920 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -7,6 +7,7 @@ #include "bh_log.h" #include "mem_alloc.h" #include "../common/wasm_runtime_common.h" +#include "../interpreter/wasm_runtime.h" #if WASM_ENABLE_SHARED_MEMORY != 0 #include "../common/wasm_shared_memory.h" #endif @@ -30,6 +31,15 @@ bh_static_assert(offsetof(WASMExecEnv, aux_stack_bottom) == 7 * sizeof(uintptr_t)); bh_static_assert(offsetof(WASMExecEnv, native_symbol) == 8 * sizeof(uintptr_t)); +bh_static_assert(offsetof(AOTModuleInstance, memories) == 1 * sizeof(uint64)); +bh_static_assert(offsetof(AOTModuleInstance, func_ptrs) == 5 * sizeof(uint64)); +bh_static_assert(offsetof(AOTModuleInstance, func_type_indexes) + == 6 * sizeof(uint64)); +bh_static_assert(offsetof(AOTModuleInstance, cur_exception) + == 13 * sizeof(uint64)); +bh_static_assert(offsetof(AOTModuleInstance, global_table_data) + == 13 * sizeof(uint64) + 128 + 11 * sizeof(uint64)); + static void set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) { @@ -114,7 +124,7 @@ init_global_data(uint8 *global_data, uint8 type, WASMValue *initial_value) break; #if WASM_ENABLE_SIMD != 0 case VALUE_TYPE_V128: - bh_memcpy_s(global_data, sizeof(V128), &initial_value->i64, + bh_memcpy_s(global_data, sizeof(V128), &initial_value->v128, sizeof(V128)); break; #endif @@ -129,14 +139,14 @@ global_instantiate(AOTModuleInstance *module_inst, AOTModule *module, { uint32 i; InitializerExpression *init_expr; - uint8 *p = (uint8 *)module_inst->global_data.ptr; + uint8 *p = module_inst->global_data; AOTImportGlobal *import_global = module->import_globals; AOTGlobal *global = module->globals; /* Initialize import global data */ for (i = 0; i < module->import_global_count; i++, import_global++) { bh_assert(import_global->data_offset - == (uint32)(p - (uint8 *)module_inst->global_data.ptr)); + == (uint32)(p - module_inst->global_data)); init_global_data(p, import_global->type, &import_global->global_data_linked); p += import_global->size; @@ -145,7 +155,7 @@ global_instantiate(AOTModuleInstance *module_inst, AOTModule *module, /* Initialize defined global data */ for (i = 0; i < module->global_count; i++, global++) { bh_assert(global->data_offset - == (uint32)(p - (uint8 *)module_inst->global_data.ptr)); + == (uint32)(p - module_inst->global_data)); init_expr = &global->init_expr; switch (init_expr->init_expr_type) { case INIT_EXPR_TYPE_GET_GLOBAL: @@ -177,39 +187,26 @@ global_instantiate(AOTModuleInstance *module_inst, AOTModule *module, } bh_assert(module_inst->global_data_size - == (uint32)(p - (uint8 *)module_inst->global_data.ptr)); + == (uint32)(p - module_inst->global_data)); return true; } -AOTTableInstance * -aot_next_tbl_inst(const AOTTableInstance *tbl_inst) -{ - uint32 offset = offsetof(AOTTableInstance, data); - offset += tbl_inst->max_size * sizeof(uint32); - return (AOTTableInstance *)((uint8 *)tbl_inst + offset); -} - -static inline AOTTableInstance * -aot_get_table_inst(const AOTModuleInstance *module_inst, uint32 tbl_idx) -{ - uint32 i = 0; - AOTTableInstance *tbl_inst = (AOTTableInstance *)module_inst->tables.ptr; - - while (i != tbl_idx) { - tbl_inst = aot_next_tbl_inst(tbl_inst); - ++i; - } - - return tbl_inst; -} - static bool -table_instantiate(AOTModuleInstance *module_inst, AOTModule *module, - char *error_buf, uint32 error_buf_size) +tables_instantiate(AOTModuleInstance *module_inst, AOTModule *module, + AOTTableInstance *first_tbl_inst, char *error_buf, + uint32 error_buf_size) { uint32 i, global_index, global_data_offset, base_offset, length; + uint64 total_size; AOTTableInitData *table_seg; - AOTTableInstance *tbl_inst = (AOTTableInstance *)module_inst->tables.ptr; + AOTTableInstance *tbl_inst = first_tbl_inst; + + total_size = (uint64)sizeof(WASMTableInstance *) * module_inst->table_count; + if (total_size > 0 + && !(module_inst->tables = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return false; + } /* * treat import table like a local one until we enable module linking @@ -219,15 +216,22 @@ table_instantiate(AOTModuleInstance *module_inst, AOTModule *module, if (i < module->import_table_count) { AOTImportTable *import_table = module->import_tables + i; tbl_inst->cur_size = import_table->table_init_size; - tbl_inst->max_size = aot_get_imp_tbl_data_slots(import_table); + tbl_inst->max_size = + aot_get_imp_tbl_data_slots(import_table, false); } else { AOTTable *table = module->tables + (i - module->import_table_count); tbl_inst->cur_size = table->table_init_size; - tbl_inst->max_size = aot_get_tbl_data_slots(table); + tbl_inst->max_size = aot_get_tbl_data_slots(table, false); } - tbl_inst = aot_next_tbl_inst(tbl_inst); + /* Set all elements to -1 to mark them as uninitialized elements */ + memset(tbl_inst->elems, 0xff, sizeof(uint32) * tbl_inst->max_size); + + module_inst->tables[i] = tbl_inst; + tbl_inst = (AOTTableInstance *)((uint8 *)tbl_inst + + offsetof(AOTTableInstance, elems) + + sizeof(uint32) * tbl_inst->max_size); } /* fill table with element segment content */ @@ -241,7 +245,7 @@ table_instantiate(AOTModuleInstance *module_inst, AOTModule *module, bh_assert(table_seg->table_index < module_inst->table_count); - tbl_inst = aot_get_table_inst(module_inst, table_seg->table_index); + tbl_inst = module_inst->tables[table_seg->table_index]; bh_assert(tbl_inst); #if WASM_ENABLE_REF_TYPES != 0 @@ -274,8 +278,8 @@ table_instantiate(AOTModuleInstance *module_inst, AOTModule *module, module->globals[global_index - module->import_global_count] .data_offset; - base_offset = *(uint32 *)((uint8 *)module_inst->global_data.ptr - + global_data_offset); + base_offset = + *(uint32 *)(module_inst->global_data + global_data_offset); } else base_offset = (uint32)table_seg->offset.u.i32; @@ -310,7 +314,7 @@ table_instantiate(AOTModuleInstance *module_inst, AOTModule *module, * Check function index in the current module inst for now. * will check the linked table inst owner in future */ - bh_memcpy_s((uint32 *)tbl_inst->data + base_offset, + bh_memcpy_s(tbl_inst->elems + base_offset, (tbl_inst->max_size - base_offset) * sizeof(uint32), table_seg->func_indexes, length * sizeof(uint32)); } @@ -325,12 +329,12 @@ memories_deinstantiate(AOTModuleInstance *module_inst) AOTMemoryInstance *memory_inst; for (i = 0; i < module_inst->memory_count; i++) { - memory_inst = ((AOTMemoryInstance **)module_inst->memories.ptr)[i]; + memory_inst = module_inst->memories[i]; if (memory_inst) { #if WASM_ENABLE_SHARED_MEMORY != 0 if (memory_inst->is_shared) { int32 ref_count = shared_memory_dec_reference( - (WASMModuleCommon *)module_inst->aot_module.ptr); + (WASMModuleCommon *)module_inst->module); bh_assert(ref_count >= 0); /* if the reference count is not zero, @@ -339,27 +343,26 @@ memories_deinstantiate(AOTModuleInstance *module_inst) continue; } #endif - if (memory_inst->heap_handle.ptr) { - mem_allocator_destroy(memory_inst->heap_handle.ptr); - wasm_runtime_free(memory_inst->heap_handle.ptr); + if (memory_inst->heap_handle) { + mem_allocator_destroy(memory_inst->heap_handle); + wasm_runtime_free(memory_inst->heap_handle); } - if (memory_inst->memory_data.ptr) { + if (memory_inst->memory_data) { #ifndef OS_ENABLE_HW_BOUND_CHECK - wasm_runtime_free(memory_inst->memory_data.ptr); + wasm_runtime_free(memory_inst->memory_data); #else #ifdef BH_PLATFORM_WINDOWS - os_mem_decommit(memory_inst->memory_data.ptr, + os_mem_decommit(memory_inst->memory_data, memory_inst->num_bytes_per_page * memory_inst->cur_page_count); #endif - os_munmap((uint8 *)memory_inst->memory_data.ptr, - 8 * (uint64)BH_GB); + os_munmap(memory_inst->memory_data, 8 * (uint64)BH_GB); #endif } } } - wasm_runtime_free(module_inst->memories.ptr); + wasm_runtime_free(module_inst->memories); } static AOTMemoryInstance * @@ -461,7 +464,7 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, /* Adjust __heap_base global value */ global_idx = module->aux_heap_base_global_index - module->import_global_count; - global_addr = (uint8 *)module_inst->global_data.ptr + global_addr = module_inst->global_data + module->globals[global_idx].data_offset; *(uint32 *)global_addr = aux_heap_base; LOG_VERBOSE("Reset __heap_base global to %u", aux_heap_base); @@ -557,12 +560,12 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, memory_inst->memory_data_size = (uint32)total_size; /* Init memory info */ - memory_inst->memory_data.ptr = p; - memory_inst->memory_data_end.ptr = p + (uint32)total_size; + memory_inst->memory_data = p; + memory_inst->memory_data_end = p + (uint32)total_size; /* Initialize heap info */ - memory_inst->heap_data.ptr = p + heap_offset; - memory_inst->heap_data_end.ptr = p + heap_offset + heap_size; + memory_inst->heap_data = p + heap_offset; + memory_inst->heap_data_end = p + heap_offset + heap_size; if (heap_size > 0) { uint32 heap_struct_size = mem_allocator_get_heap_struct_size(); @@ -571,10 +574,10 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, goto fail1; } - memory_inst->heap_handle.ptr = heap_handle; + memory_inst->heap_handle = heap_handle; if (!mem_allocator_create_with_struct_and_pool( - heap_handle, heap_struct_size, memory_inst->heap_data.ptr, + heap_handle, heap_struct_size, memory_inst->heap_data, heap_size)) { set_error_buf(error_buf, error_buf_size, "init app heap failed"); goto fail2; @@ -614,31 +617,31 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, #if WASM_ENABLE_SHARED_MEMORY != 0 fail3: if (heap_size > 0) - mem_allocator_destroy(memory_inst->heap_handle.ptr); + mem_allocator_destroy(memory_inst->heap_handle); #endif fail2: if (heap_size > 0) - wasm_runtime_free(memory_inst->heap_handle.ptr); + wasm_runtime_free(memory_inst->heap_handle); fail1: #ifndef OS_ENABLE_HW_BOUND_CHECK - if (memory_inst->memory_data.ptr) - wasm_runtime_free(memory_inst->memory_data.ptr); + if (memory_inst->memory_data) + wasm_runtime_free(memory_inst->memory_data); #else #ifdef BH_PLATFORM_WINDOWS - if (memory_inst->memory_data.ptr) + if (memory_inst->memory_data) os_mem_decommit(p, total_size); #endif os_munmap(mapped_mem, map_size); #endif - memory_inst->memory_data.ptr = NULL; + memory_inst->memory_data = NULL; return NULL; } static AOTMemoryInstance * aot_get_default_memory(AOTModuleInstance *module_inst) { - if (module_inst->memories.ptr) - return ((AOTMemoryInstance **)module_inst->memories.ptr)[0]; + if (module_inst->memories) + return module_inst->memories[0]; else return NULL; } @@ -654,8 +657,8 @@ memories_instantiate(AOTModuleInstance *module_inst, AOTModule *module, uint64 total_size; module_inst->memory_count = memory_count; - total_size = sizeof(AOTPointer) * (uint64)memory_count; - if (!(module_inst->memories.ptr = + total_size = sizeof(AOTMemoryInstance *) * (uint64)memory_count; + if (!(module_inst->memories = runtime_malloc(total_size, error_buf, error_buf_size))) { return false; } @@ -669,7 +672,7 @@ memories_instantiate(AOTModuleInstance *module_inst, AOTModule *module, return false; } - ((AOTMemoryInstance **)module_inst->memories.ptr)[i] = memory_inst; + module_inst->memories[i] = memory_inst; } /* Get default memory instance */ @@ -707,15 +710,15 @@ memories_instantiate(AOTModuleInstance *module_inst, AOTModule *module, module->globals[global_index - module->import_global_count] .data_offset; - base_offset = *(uint32 *)((uint8 *)module_inst->global_data.ptr - + global_data_offset); + base_offset = + *(uint32 *)(module_inst->global_data + global_data_offset); } else { base_offset = (uint32)data_seg->offset.u.i32; } /* Copy memory data */ - bh_assert(memory_inst->memory_data.ptr + bh_assert(memory_inst->memory_data || memory_inst->memory_data_size == 0); /* Check memory data */ @@ -748,8 +751,8 @@ memories_instantiate(AOTModuleInstance *module_inst, AOTModule *module, return false; } - if (memory_inst->memory_data.ptr) { - bh_memcpy_s((uint8 *)memory_inst->memory_data.ptr + base_offset, + if (memory_inst->memory_data) { + bh_memcpy_s((uint8 *)memory_inst->memory_data + base_offset, memory_inst->memory_data_size - base_offset, data_seg->bytes, length); } @@ -771,13 +774,13 @@ init_func_ptrs(AOTModuleInstance *module_inst, AOTModule *module, return true; /* Allocate memory */ - if (!(module_inst->func_ptrs.ptr = + if (!(module_inst->func_ptrs = runtime_malloc(total_size, error_buf, error_buf_size))) { return false; } /* Set import function pointers */ - func_ptrs = (void **)module_inst->func_ptrs.ptr; + func_ptrs = (void **)module_inst->func_ptrs; for (i = 0; i < module->import_func_count; i++, func_ptrs++) { *func_ptrs = (void *)module->import_funcs[i].func_ptr_linked; if (!*func_ptrs) { @@ -807,13 +810,13 @@ init_func_type_indexes(AOTModuleInstance *module_inst, AOTModule *module, return true; /* Allocate memory */ - if (!(module_inst->func_type_indexes.ptr = + if (!(module_inst->func_type_indexes = runtime_malloc(total_size, error_buf, error_buf_size))) { return false; } /* Set import function type indexes */ - func_type_index = (uint32 *)module_inst->func_type_indexes.ptr; + func_type_index = module_inst->func_type_indexes; for (i = 0; i < module->import_func_count; i++, func_type_index++) *func_type_index = module->import_funcs[i].func_type_index; @@ -835,10 +838,10 @@ create_export_funcs(AOTModuleInstance *module_inst, AOTModule *module, /* Allocate memory */ size = sizeof(AOTFunctionInstance) * (uint64)module_inst->export_func_count; - if (!(module_inst->export_funcs.ptr = export_func = - runtime_malloc(size, error_buf, error_buf_size))) { + if (!(export_func = runtime_malloc(size, error_buf, error_buf_size))) { return false; } + module_inst->export_functions = (void *)export_func; for (i = 0; i < module->export_count; i++) { if (exports[i].kind == EXPORT_KIND_FUNC) { @@ -883,10 +886,10 @@ create_exports(AOTModuleInstance *module_inst, AOTModule *module, module_inst->export_global_count++; break; case EXPORT_KIND_TABLE: - module_inst->export_tab_count++; + module_inst->export_table_count++; break; case EXPORT_KIND_MEMORY: - module_inst->export_mem_count++; + module_inst->export_memory_count++; break; default: return false; @@ -931,7 +934,7 @@ execute_post_inst_function(AOTModuleInstance *module_inst) static bool execute_start_function(AOTModuleInstance *module_inst) { - AOTModule *module = (AOTModule *)module_inst->aot_module.ptr; + AOTModule *module = (AOTModule *)module_inst->module; WASMExecEnv *exec_env; typedef void (*F)(WASMExecEnv *); union { @@ -998,16 +1001,17 @@ aot_instantiate(AOTModule *module, bool is_sub_inst, uint32 stack_size, * calculate size of table data */ for (i = 0; i != module->import_table_count; ++i) { - table_size += offsetof(AOTTableInstance, data); - table_size += - (uint64)sizeof(uint32) - * (uint64)aot_get_imp_tbl_data_slots(module->import_tables + i); + table_size += offsetof(AOTTableInstance, elems); + table_size += (uint64)sizeof(uint32) + * (uint64)aot_get_imp_tbl_data_slots( + module->import_tables + i, false); } for (i = 0; i != module->table_count; ++i) { - table_size += offsetof(AOTTableInstance, data); - table_size += (uint64)sizeof(uint32) - * (uint64)aot_get_tbl_data_slots(module->tables + i); + table_size += offsetof(AOTTableInstance, elems); + table_size += + (uint64)sizeof(uint32) + * (uint64)aot_get_tbl_data_slots(module->tables + i, false); } total_size += table_size; @@ -1018,23 +1022,21 @@ aot_instantiate(AOTModule *module, bool is_sub_inst, uint32 stack_size, } module_inst->module_type = Wasm_Module_AoT; - module_inst->aot_module.ptr = module; + module_inst->module = (void *)module; /* Initialize global info */ p = (uint8 *)module_inst + module_inst_struct_size + module_inst_mem_inst_size; - module_inst->global_data.ptr = p; + module_inst->global_data = p; module_inst->global_data_size = module->global_data_size; if (!global_instantiate(module_inst, module, error_buf, error_buf_size)) goto fail; /* Initialize table info */ p += module->global_data_size; - module_inst->tables.ptr = p; module_inst->table_count = module->table_count + module->import_table_count; - /* Set all elements to -1 to mark them as uninitialized elements */ - memset(module_inst->tables.ptr, 0xff, (uint32)table_size); - if (!table_instantiate(module_inst, module, error_buf, error_buf_size)) + if (!tables_instantiate(module_inst, module, (AOTTableInstance *)p, + error_buf, error_buf_size)) goto fail; /* Initialize memory space */ @@ -1082,14 +1084,14 @@ aot_instantiate(AOTModule *module, bool is_sub_inst, uint32 stack_size, #if WASM_ENABLE_PERF_PROFILING != 0 total_size = (uint64)sizeof(AOTFuncPerfProfInfo) * (module->import_func_count + module->func_count); - if (!(module_inst->func_perf_profilings.ptr = + if (!(module_inst->func_perf_profilings = runtime_malloc(total_size, error_buf, error_buf_size))) { goto fail; } #endif #if WASM_ENABLE_DUMP_CALL_STACK != 0 - if (!(module_inst->frames.ptr = + if (!(module_inst->frames = runtime_malloc(sizeof(Vector), error_buf, error_buf_size))) { goto fail; } @@ -1133,18 +1135,6 @@ fail: return NULL; } -bool -aot_create_exec_env_singleton(AOTModuleInstance *module_inst) -{ - WASMExecEnv *exec_env = - wasm_exec_env_create((WASMModuleInstanceCommon *)module_inst, - module_inst->default_wasm_stack_size); - if (exec_env) - module_inst->exec_env_singleton.ptr = exec_env; - - return exec_env ? true : false; -} - void aot_deinstantiate(AOTModuleInstance *module_inst, bool is_sub_inst) { @@ -1159,33 +1149,35 @@ aot_deinstantiate(AOTModuleInstance *module_inst, bool is_sub_inst) #endif #if WASM_ENABLE_PERF_PROFILING != 0 - if (module_inst->func_perf_profilings.ptr) - wasm_runtime_free(module_inst->func_perf_profilings.ptr); + if (module_inst->func_perf_profilings) + wasm_runtime_free(module_inst->func_perf_profilings); #endif #if WASM_ENABLE_DUMP_CALL_STACK != 0 - if (module_inst->frames.ptr) { - bh_vector_destroy(module_inst->frames.ptr); - wasm_runtime_free(module_inst->frames.ptr); - module_inst->frames.ptr = NULL; + if (module_inst->frames) { + bh_vector_destroy(module_inst->frames); + wasm_runtime_free(module_inst->frames); + module_inst->frames = NULL; } #endif - if (module_inst->memories.ptr) + if (module_inst->tables) + wasm_runtime_free(module_inst->tables); + + if (module_inst->memories) memories_deinstantiate(module_inst); - if (module_inst->export_funcs.ptr) - wasm_runtime_free(module_inst->export_funcs.ptr); + if (module_inst->export_functions) + wasm_runtime_free(module_inst->export_functions); - if (module_inst->func_ptrs.ptr) - wasm_runtime_free(module_inst->func_ptrs.ptr); + if (module_inst->func_ptrs) + wasm_runtime_free(module_inst->func_ptrs); - if (module_inst->func_type_indexes.ptr) - wasm_runtime_free(module_inst->func_type_indexes.ptr); + if (module_inst->func_type_indexes) + wasm_runtime_free(module_inst->func_type_indexes); - if (module_inst->exec_env_singleton.ptr) - wasm_exec_env_destroy( - (WASMExecEnv *)module_inst->exec_env_singleton.ptr); + if (module_inst->exec_env_singleton) + wasm_exec_env_destroy((WASMExecEnv *)module_inst->exec_env_singleton); wasm_runtime_free(module_inst); } @@ -1196,7 +1188,7 @@ aot_lookup_function(const AOTModuleInstance *module_inst, const char *name, { uint32 i; AOTFunctionInstance *export_funcs = - (AOTFunctionInstance *)module_inst->export_funcs.ptr; + (AOTFunctionInstance *)module_inst->export_functions; for (i = 0; i < module_inst->export_func_count; i++) if (!strcmp(export_funcs[i].func_name, name)) @@ -1207,115 +1199,6 @@ aot_lookup_function(const AOTModuleInstance *module_inst, const char *name, #ifdef OS_ENABLE_HW_BOUND_CHECK -#ifndef BH_PLATFORM_WINDOWS -void -aot_signal_handler(WASMSignalInfo *sig_info) -{ - WASMExecEnv *exec_env_tls = sig_info->exec_env_tls; - void *sig_addr = sig_info->sig_addr; - AOTModuleInstance *module_inst; - AOTMemoryInstance *memory_inst; - WASMJmpBuf *jmpbuf_node; - uint8 *mapped_mem_start_addr = NULL; - uint8 *mapped_mem_end_addr = NULL; - uint8 *stack_min_addr; - uint32 page_size; - uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; - - /* Check whether current thread is running aot function */ - if (exec_env_tls && exec_env_tls->handle == os_self_thread() - && (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) { - /* Get mapped mem info of current instance */ - module_inst = (AOTModuleInstance *)exec_env_tls->module_inst; - /* Get the default memory instance */ - memory_inst = aot_get_default_memory(module_inst); - if (memory_inst) { - mapped_mem_start_addr = (uint8 *)memory_inst->memory_data.ptr; - mapped_mem_end_addr = - (uint8 *)memory_inst->memory_data.ptr + 8 * (uint64)BH_GB; - } - - /* Get stack info of current thread */ - page_size = os_getpagesize(); - stack_min_addr = os_thread_get_stack_boundary(); - - if (memory_inst - && (mapped_mem_start_addr <= (uint8 *)sig_addr - && (uint8 *)sig_addr < mapped_mem_end_addr)) { - /* The address which causes segmentation fault is inside - the memory instance's guard regions */ - aot_set_exception_with_id(module_inst, - EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS); - os_longjmp(jmpbuf_node->jmpbuf, 1); - } - else if (stack_min_addr - page_size <= (uint8 *)sig_addr - && (uint8 *)sig_addr - < stack_min_addr + page_size * guard_page_count) { - /* The address which causes segmentation fault is inside - native thread's guard page */ - aot_set_exception_with_id(module_inst, EXCE_NATIVE_STACK_OVERFLOW); - os_longjmp(jmpbuf_node->jmpbuf, 1); - } - } -} -#else /* else of BH_PLATFORM_WINDOWS */ -LONG -aot_exception_handler(WASMSignalInfo *sig_info) -{ - WASMExecEnv *exec_env_tls = sig_info->exec_env_tls; - EXCEPTION_POINTERS *exce_info = sig_info->exce_info; - PEXCEPTION_RECORD ExceptionRecord = exce_info->ExceptionRecord; - uint8 *sig_addr = (uint8 *)ExceptionRecord->ExceptionInformation[1]; - AOTModuleInstance *module_inst; - AOTMemoryInstance *memory_inst; - WASMJmpBuf *jmpbuf_node; - uint8 *mapped_mem_start_addr = NULL; - uint8 *mapped_mem_end_addr = NULL; - uint32 page_size = os_getpagesize(); - - if (exec_env_tls && exec_env_tls->handle == os_self_thread() - && (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) { - module_inst = (AOTModuleInstance *)exec_env_tls->module_inst; - if (ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { - /* Get the default memory instance */ - memory_inst = aot_get_default_memory(module_inst); - if (memory_inst) { - mapped_mem_start_addr = (uint8 *)memory_inst->memory_data.ptr; - mapped_mem_end_addr = - (uint8 *)memory_inst->memory_data.ptr + 8 * (uint64)BH_GB; - if (mapped_mem_start_addr <= (uint8 *)sig_addr - && (uint8 *)sig_addr < mapped_mem_end_addr) { - /* The address which causes segmentation fault is inside - aot instance's guard regions. - Set exception and let the aot func continue to run, when - the aot func returns, the caller will check whether the - exception is thrown and return to runtime. */ - aot_set_exception_with_id(module_inst, - EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS); - /* Skip current instruction */ - exce_info->ContextRecord->Rip++; - return EXCEPTION_CONTINUE_EXECUTION; - } - } - } - else if (ExceptionRecord->ExceptionCode == EXCEPTION_STACK_OVERFLOW) { - /* Set stack overflow exception and let the aot func continue - to run, when the aot func returns, the caller will check - whether the exception is thrown and return to runtime, and - the damaged stack will be recovered by _resetstkoflw(). */ - aot_set_exception_with_id(module_inst, EXCE_NATIVE_STACK_OVERFLOW); - return EXCEPTION_CONTINUE_EXECUTION; - } - } - - os_printf("Unhandled exception thrown: exception code: 0x%lx, " - "exception address: %p, exception information: %p\n", - ExceptionRecord->ExceptionCode, ExceptionRecord->ExceptionAddress, - sig_addr); - return EXCEPTION_CONTINUE_SEARCH; -} -#endif /* end of BH_PLATFORM_WINDOWS */ - static bool invoke_native_with_hw_bound_check(WASMExecEnv *exec_env, void *func_ptr, const WASMType *func_type, @@ -1438,17 +1321,6 @@ aot_call_function(WASMExecEnv *exec_env, AOTFunctionInstance *function, } argc = func_type->param_cell_num; -#if WASM_ENABLE_LAZY_JIT != 0 - if (!function->u.func.func_ptr) { - AOTModule *aot_module = (AOTModule *)module_inst->aot_module.ptr; - if (!(function->u.func.func_ptr = - aot_lookup_orcjit_func(aot_module->comp_ctx->orc_lazyjit, - module_inst, function->func_index))) { - return false; - } - } -#endif - /* set thread handle and stack boundary */ wasm_exec_env_set_thread_info(exec_env); @@ -1619,84 +1491,19 @@ aot_create_exec_env_and_call_function(AOTModuleInstance *module_inst, void aot_set_exception(AOTModuleInstance *module_inst, const char *exception) { - if (exception) - snprintf(module_inst->cur_exception, sizeof(module_inst->cur_exception), - "Exception: %s", exception); - else - module_inst->cur_exception[0] = '\0'; + wasm_set_exception(module_inst, exception); } void aot_set_exception_with_id(AOTModuleInstance *module_inst, uint32 id) { - switch (id) { - case EXCE_UNREACHABLE: - aot_set_exception(module_inst, "unreachable"); - break; - case EXCE_OUT_OF_MEMORY: - aot_set_exception(module_inst, "allocate memory failed"); - break; - case EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS: - aot_set_exception(module_inst, "out of bounds memory access"); - break; - case EXCE_INTEGER_OVERFLOW: - aot_set_exception(module_inst, "integer overflow"); - break; - case EXCE_INTEGER_DIVIDE_BY_ZERO: - aot_set_exception(module_inst, "integer divide by zero"); - break; - case EXCE_INVALID_CONVERSION_TO_INTEGER: - aot_set_exception(module_inst, "invalid conversion to integer"); - break; - case EXCE_INVALID_FUNCTION_TYPE_INDEX: - aot_set_exception(module_inst, "indirect call type mismatch"); - break; - case EXCE_INVALID_FUNCTION_INDEX: - aot_set_exception(module_inst, "invalid function index"); - break; - case EXCE_UNDEFINED_ELEMENT: - aot_set_exception(module_inst, "undefined element"); - break; - case EXCE_UNINITIALIZED_ELEMENT: - aot_set_exception(module_inst, "uninitialized element"); - break; - case EXCE_CALL_UNLINKED_IMPORT_FUNC: - aot_set_exception(module_inst, - "failed to call unlinked import function"); - break; - case EXCE_NATIVE_STACK_OVERFLOW: - aot_set_exception(module_inst, "native stack overflow"); - break; - case EXCE_UNALIGNED_ATOMIC: - aot_set_exception(module_inst, "unaligned atomic"); - break; - case EXCE_AUX_STACK_OVERFLOW: - aot_set_exception(module_inst, "wasm auxiliary stack overflow"); - break; - case EXCE_AUX_STACK_UNDERFLOW: - aot_set_exception(module_inst, "wasm auxiliary stack underflow"); - break; - case EXCE_OUT_OF_BOUNDS_TABLE_ACCESS: - aot_set_exception(module_inst, "out of bounds table access"); - break; - default: - break; - } + wasm_set_exception_with_id(module_inst, id); } const char * aot_get_exception(AOTModuleInstance *module_inst) { - if (module_inst->cur_exception[0] == '\0') - return NULL; - else - return module_inst->cur_exception; -} - -void -aot_clear_exception(AOTModuleInstance *module_inst) -{ - module_inst->cur_exception[0] = '\0'; + return wasm_get_exception(module_inst); } static bool @@ -1774,7 +1581,7 @@ aot_module_malloc(AOTModuleInstance *module_inst, uint32 size, void **p_native_addr) { AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - AOTModule *module = (AOTModule *)module_inst->aot_module.ptr; + AOTModule *module = (AOTModule *)module_inst->module; uint8 *addr = NULL; uint32 offset = 0; @@ -1783,8 +1590,8 @@ aot_module_malloc(AOTModuleInstance *module_inst, uint32 size, return 0; } - if (memory_inst->heap_handle.ptr) { - addr = mem_allocator_malloc(memory_inst->heap_handle.ptr, size); + if (memory_inst->heap_handle) { + addr = mem_allocator_malloc(memory_inst->heap_handle, size); } else if (module->malloc_func_index != (uint32)-1 && module->free_func_index != (uint32)-1) { @@ -1812,12 +1619,12 @@ aot_module_malloc(AOTModuleInstance *module_inst, uint32 size, size, &offset)) { return 0; } - addr = offset ? (uint8 *)memory_inst->memory_data.ptr + offset : NULL; + addr = offset ? (uint8 *)memory_inst->memory_data + offset : NULL; } if (!addr) { - if (memory_inst->heap_handle.ptr - && mem_allocator_is_heap_corrupted(memory_inst->heap_handle.ptr)) { + if (memory_inst->heap_handle + && mem_allocator_is_heap_corrupted(memory_inst->heap_handle)) { wasm_runtime_show_app_heap_corrupted_prompt(); aot_set_exception(module_inst, "app heap corrupted"); } @@ -1828,7 +1635,7 @@ aot_module_malloc(AOTModuleInstance *module_inst, uint32 size, } if (p_native_addr) *p_native_addr = addr; - return (uint32)(addr - (uint8 *)memory_inst->memory_data.ptr); + return (uint32)(addr - memory_inst->memory_data); } uint32 @@ -1843,17 +1650,17 @@ aot_module_realloc(AOTModuleInstance *module_inst, uint32 ptr, uint32 size, return 0; } - if (memory_inst->heap_handle.ptr) { + if (memory_inst->heap_handle) { addr = mem_allocator_realloc( - memory_inst->heap_handle.ptr, - ptr ? (uint8 *)memory_inst->memory_data.ptr + ptr : NULL, size); + memory_inst->heap_handle, + ptr ? memory_inst->memory_data + ptr : NULL, size); } /* Only support realloc in WAMR's app heap */ if (!addr) { - if (memory_inst->heap_handle.ptr - && mem_allocator_is_heap_corrupted(memory_inst->heap_handle.ptr)) { + if (memory_inst->heap_handle + && mem_allocator_is_heap_corrupted(memory_inst->heap_handle)) { aot_set_exception(module_inst, "app heap corrupted"); } else { @@ -1864,30 +1671,29 @@ aot_module_realloc(AOTModuleInstance *module_inst, uint32 ptr, uint32 size, if (p_native_addr) *p_native_addr = addr; - return (uint32)(addr - (uint8 *)memory_inst->memory_data.ptr); + return (uint32)(addr - memory_inst->memory_data); } void aot_module_free(AOTModuleInstance *module_inst, uint32 ptr) { AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - AOTModule *module = (AOTModule *)module_inst->aot_module.ptr; + AOTModule *module = (AOTModule *)module_inst->module; if (!memory_inst) { return; } if (ptr) { - uint8 *addr = (uint8 *)memory_inst->memory_data.ptr + ptr; - if (memory_inst->heap_handle.ptr - && (uint8 *)memory_inst->heap_data.ptr < addr - && addr < (uint8 *)memory_inst->heap_data_end.ptr) { - mem_allocator_free(memory_inst->heap_handle.ptr, addr); + uint8 *addr = memory_inst->memory_data + ptr; + if (memory_inst->heap_handle && memory_inst->heap_data < addr + && addr < memory_inst->heap_data_end) { + mem_allocator_free(memory_inst->heap_handle, addr); } else if (module->malloc_func_index != (uint32)-1 && module->free_func_index != (uint32)-1 - && (uint8 *)memory_inst->memory_data.ptr <= addr - && addr < (uint8 *)memory_inst->memory_data_end.ptr) { + && memory_inst->memory_data <= addr + && addr < memory_inst->memory_data_end) { AOTFunctionInstance *free_func; char *free_func_name; @@ -1917,331 +1723,18 @@ aot_module_dup_data(AOTModuleInstance *module_inst, const char *src, aot_module_malloc(module_inst, size, (void **)&buffer); if (buffer_offset != 0) { - buffer = aot_addr_app_to_native(module_inst, buffer_offset); + buffer = wasm_runtime_addr_app_to_native( + (WASMModuleInstanceCommon *)module_inst, buffer_offset); bh_memcpy_s(buffer, size, src, size); } return buffer_offset; } -bool -aot_validate_app_addr(AOTModuleInstance *module_inst, uint32 app_offset, - uint32 size) -{ - AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - - if (!memory_inst) { - goto fail; - } - - /* integer overflow check */ - if (app_offset > UINT32_MAX - size) { - goto fail; - } - - if (app_offset + size <= memory_inst->memory_data_size) { - return true; - } -fail: - aot_set_exception(module_inst, "out of bounds memory access"); - return false; -} - -bool -aot_validate_native_addr(AOTModuleInstance *module_inst, void *native_ptr, - uint32 size) -{ - AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - uint8 *addr = (uint8 *)native_ptr; - - if (!memory_inst) { - goto fail; - } - - /* integer overflow check */ - if ((uintptr_t)addr > UINTPTR_MAX - size) { - goto fail; - } - - if ((uint8 *)memory_inst->memory_data.ptr <= addr - && addr + size <= (uint8 *)memory_inst->memory_data_end.ptr) - return true; -fail: - aot_set_exception(module_inst, "out of bounds memory access"); - return false; -} - -void * -aot_addr_app_to_native(AOTModuleInstance *module_inst, uint32 app_offset) -{ - AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - uint8 *addr; - - if (!memory_inst) { - return NULL; - } - - addr = (uint8 *)memory_inst->memory_data.ptr + app_offset; - - if ((uint8 *)memory_inst->memory_data.ptr <= addr - && addr < (uint8 *)memory_inst->memory_data_end.ptr) - return addr; - return NULL; -} - -uint32 -aot_addr_native_to_app(AOTModuleInstance *module_inst, void *native_ptr) -{ - AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - uint8 *addr = (uint8 *)native_ptr; - - if (!memory_inst) { - return 0; - } - - if ((uint8 *)memory_inst->memory_data.ptr <= addr - && addr < (uint8 *)memory_inst->memory_data_end.ptr) - return (uint32)(addr - (uint8 *)memory_inst->memory_data.ptr); - return 0; -} - -bool -aot_get_app_addr_range(AOTModuleInstance *module_inst, uint32 app_offset, - uint32 *p_app_start_offset, uint32 *p_app_end_offset) -{ - AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - uint32 memory_data_size; - - if (!memory_inst) { - return false; - } - - memory_data_size = memory_inst->memory_data_size; - - if (app_offset < memory_data_size) { - if (p_app_start_offset) - *p_app_start_offset = 0; - if (p_app_end_offset) - *p_app_end_offset = memory_data_size; - return true; - } - return false; -} - -bool -aot_get_native_addr_range(AOTModuleInstance *module_inst, uint8 *native_ptr, - uint8 **p_native_start_addr, - uint8 **p_native_end_addr) -{ - AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - uint8 *addr = (uint8 *)native_ptr; - - if (!memory_inst) { - return false; - } - - if ((uint8 *)memory_inst->memory_data.ptr <= addr - && addr < (uint8 *)memory_inst->memory_data_end.ptr) { - if (p_native_start_addr) - *p_native_start_addr = (uint8 *)memory_inst->memory_data.ptr; - if (p_native_end_addr) - *p_native_end_addr = (uint8 *)memory_inst->memory_data_end.ptr; - return true; - } - return false; -} - -#ifndef OS_ENABLE_HW_BOUND_CHECK bool aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) { - AOTMemoryInstance *memory = aot_get_default_memory(module_inst); - uint8 *memory_data_old, *memory_data_new, *heap_data_old; - uint32 num_bytes_per_page, heap_size, total_size_old; - uint32 cur_page_count, max_page_count, total_page_count; - uint64 total_size_new; - bool ret = true; - - if (!memory) - return false; - - heap_data_old = (uint8 *)memory->heap_data.ptr; - heap_size = (uint32)((uint8 *)memory->heap_data_end.ptr - - (uint8 *)memory->heap_data.ptr); - - memory_data_old = (uint8 *)memory->memory_data.ptr; - total_size_old = - (uint32)((uint8 *)memory->memory_data_end.ptr - memory_data_old); - - num_bytes_per_page = memory->num_bytes_per_page; - cur_page_count = memory->cur_page_count; - max_page_count = memory->max_page_count; - total_page_count = inc_page_count + cur_page_count; - total_size_new = num_bytes_per_page * (uint64)total_page_count; - - if (inc_page_count <= 0) - /* No need to enlarge memory */ - return true; - - if (total_page_count < cur_page_count /* integer overflow */ - || total_page_count > max_page_count) { - return false; - } - - bh_assert(total_size_new <= 4 * (uint64)BH_GB); - if (total_size_new > UINT32_MAX) { - /* Resize to 1 page with size 4G-1 */ - num_bytes_per_page = UINT32_MAX; - total_page_count = max_page_count = 1; - total_size_new = UINT32_MAX; - } - -#if WASM_ENABLE_SHARED_MEMORY != 0 - if (memory->is_shared) { - memory->num_bytes_per_page = num_bytes_per_page; - memory->cur_page_count = total_page_count; - memory->max_page_count = max_page_count; - /* No need to update memory->memory_data_size as it is - initialized with the maximum memory data size for - shared memory */ - return true; - } -#endif - - if (heap_size > 0) { - if (mem_allocator_is_heap_corrupted(memory->heap_handle.ptr)) { - wasm_runtime_show_app_heap_corrupted_prompt(); - return false; - } - } - - if (!(memory_data_new = - wasm_runtime_realloc(memory_data_old, (uint32)total_size_new))) { - if (!(memory_data_new = wasm_runtime_malloc((uint32)total_size_new))) { - return false; - } - if (memory_data_old) { - bh_memcpy_s(memory_data_new, (uint32)total_size_new, - memory_data_old, total_size_old); - wasm_runtime_free(memory_data_old); - } - } - - memset(memory_data_new + total_size_old, 0, - (uint32)total_size_new - total_size_old); - - if (heap_size > 0) { - if (mem_allocator_migrate(memory->heap_handle.ptr, - (char *)heap_data_old - + (memory_data_new - memory_data_old), - heap_size) - != 0) { - /* Don't return here as memory->memory_data is obsolete and - must be updated to be correctly used later. */ - ret = false; - } - } - - memory->heap_data.ptr = memory_data_new + (heap_data_old - memory_data_old); - memory->heap_data_end.ptr = (uint8 *)memory->heap_data.ptr + heap_size; - - memory->num_bytes_per_page = num_bytes_per_page; - memory->cur_page_count = total_page_count; - memory->max_page_count = max_page_count; - - memory->memory_data.ptr = memory_data_new; - memory->memory_data_end.ptr = memory_data_new + (uint32)total_size_new; - memory->memory_data_size = (uint32)total_size_new; - -#if UINTPTR_MAX == UINT64_MAX - memory->mem_bound_check_1byte.u64 = total_size_new - 1; - memory->mem_bound_check_2bytes.u64 = total_size_new - 2; - memory->mem_bound_check_4bytes.u64 = total_size_new - 4; - memory->mem_bound_check_8bytes.u64 = total_size_new - 8; - memory->mem_bound_check_16bytes.u64 = total_size_new - 16; -#else - memory->mem_bound_check_1byte.u32[0] = (uint32)total_size_new - 1; - memory->mem_bound_check_2bytes.u32[0] = (uint32)total_size_new - 2; - memory->mem_bound_check_4bytes.u32[0] = (uint32)total_size_new - 4; - memory->mem_bound_check_8bytes.u32[0] = (uint32)total_size_new - 8; - memory->mem_bound_check_16bytes.u32[0] = (uint32)total_size_new - 16; -#endif - - return ret; + return wasm_enlarge_memory(module_inst, inc_page_count); } -#else /* else of OS_ENABLE_HW_BOUND_CHECK */ -bool -aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) -{ - AOTMemoryInstance *memory = aot_get_default_memory(module_inst); - uint32 num_bytes_per_page, total_size_old; - uint32 cur_page_count, max_page_count, total_page_count; - uint64 total_size_new; - - if (!memory) - return false; - - num_bytes_per_page = memory->num_bytes_per_page; - cur_page_count = memory->cur_page_count; - max_page_count = memory->max_page_count; - total_size_old = num_bytes_per_page * cur_page_count; - total_page_count = inc_page_count + cur_page_count; - total_size_new = num_bytes_per_page * (uint64)total_page_count; - - if (inc_page_count <= 0) - /* No need to enlarge memory */ - return true; - - if (total_page_count < cur_page_count /* integer overflow */ - || total_page_count > max_page_count) { - return false; - } - - bh_assert(total_size_new <= 4 * (uint64)BH_GB); - if (total_size_new > UINT32_MAX) { - /* Resize to 1 page with size 4G-1 */ - num_bytes_per_page = UINT32_MAX; - total_page_count = max_page_count = 1; - total_size_new = UINT32_MAX; - } - -#ifdef BH_PLATFORM_WINDOWS - if (!os_mem_commit(memory->memory_data_end.ptr, - (uint32)total_size_new - total_size_old, - MMAP_PROT_READ | MMAP_PROT_WRITE)) { - return false; - } -#endif - - if (os_mprotect(memory->memory_data_end.ptr, - (uint32)total_size_new - total_size_old, - MMAP_PROT_READ | MMAP_PROT_WRITE) - != 0) { -#ifdef BH_PLATFORM_WINDOWS - os_mem_decommit(memory->memory_data_end.ptr, - (uint32)total_size_new - total_size_old); -#endif - return false; - } - - /* The increased pages are filled with zero by the OS when os_mmap, - no need to memset it again here */ - - memory->num_bytes_per_page = num_bytes_per_page; - memory->cur_page_count = total_page_count; - memory->max_page_count = max_page_count; - memory->memory_data_size = (uint32)total_size_new; - memory->memory_data_end.ptr = - (uint8 *)memory->memory_data.ptr + (uint32)total_size_new; - - memory->mem_bound_check_1byte.u64 = total_size_new - 1; - memory->mem_bound_check_2bytes.u64 = total_size_new - 2; - memory->mem_bound_check_4bytes.u64 = total_size_new - 4; - memory->mem_bound_check_8bytes.u64 = total_size_new - 8; - memory->mem_bound_check_16bytes.u64 = total_size_new - 16; - - return true; -} -#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ bool aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc, @@ -2249,11 +1742,11 @@ aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc, { AOTModuleInstance *module_inst = (AOTModuleInstance *)wasm_runtime_get_module_inst(exec_env); - AOTModule *aot_module = (AOTModule *)module_inst->aot_module.ptr; - uint32 *func_type_indexes = (uint32 *)module_inst->func_type_indexes.ptr; + AOTModule *aot_module = (AOTModule *)module_inst->module; + uint32 *func_type_indexes = module_inst->func_type_indexes; uint32 func_type_idx = func_type_indexes[func_idx]; AOTFuncType *func_type = aot_module->func_types[func_type_idx]; - void **func_ptrs = (void **)module_inst->func_ptrs.ptr; + void **func_ptrs = module_inst->func_ptrs; void *func_ptr = func_ptrs[func_idx]; AOTImportFunc *import_func; const char *signature; @@ -2297,11 +1790,11 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, { AOTModuleInstance *module_inst = (AOTModuleInstance *)wasm_runtime_get_module_inst(exec_env); - AOTModule *aot_module = (AOTModule *)module_inst->aot_module.ptr; - uint32 *func_type_indexes = (uint32 *)module_inst->func_type_indexes.ptr; + AOTModule *aot_module = (AOTModule *)module_inst->module; + uint32 *func_type_indexes = module_inst->func_type_indexes; AOTTableInstance *tbl_inst; AOTFuncType *func_type; - void **func_ptrs = (void **)module_inst->func_ptrs.ptr, *func_ptr; + void **func_ptrs = module_inst->func_ptrs, *func_ptr; uint32 func_type_idx, func_idx, ext_ret_count; AOTImportFunc *import_func; const char *signature = NULL; @@ -2318,7 +1811,7 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, return false; } - tbl_inst = aot_get_table_inst(module_inst, tbl_idx); + tbl_inst = module_inst->tables[tbl_idx]; bh_assert(tbl_inst); if (table_elem_idx >= tbl_inst->cur_size) { @@ -2326,8 +1819,8 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, return false; } - func_idx = ((uint32 *)tbl_inst->data)[table_elem_idx]; - if (func_idx == (uint32)-1) { + func_idx = tbl_inst->elems[table_elem_idx]; + if (func_idx == NULL_REF) { aot_set_exception_with_id(module_inst, EXCE_UNINITIALIZED_ELEMENT); return false; } @@ -2335,15 +1828,6 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, func_type_idx = func_type_indexes[func_idx]; func_type = aot_module->func_types[func_type_idx]; -#if WASM_ENABLE_LAZY_JIT != 0 - if (func_idx >= aot_module->import_func_count && !func_ptrs[func_idx]) { - if (!(func_ptr = aot_lookup_orcjit_func( - aot_module->comp_ctx->orc_lazyjit, module_inst, func_idx))) { - return false; - } - } -#endif - if (!(func_ptr = func_ptrs[func_idx])) { bh_assert(func_idx < aot_module->import_func_count); import_func = aot_module->import_funcs + func_idx; @@ -2455,54 +1939,13 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, } } -/** - * Check whether the app address and the buf is inside the linear memory, - * and convert the app address into native address - */ bool aot_check_app_addr_and_convert(AOTModuleInstance *module_inst, bool is_str, uint32 app_buf_addr, uint32 app_buf_size, void **p_native_addr) { - AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - uint8 *native_addr; - - if (!memory_inst) { - goto fail; - } - - native_addr = (uint8 *)memory_inst->memory_data.ptr + app_buf_addr; - - /* No need to check the app_offset and buf_size if memory access - boundary check with hardware trap is enabled */ -#ifndef OS_ENABLE_HW_BOUND_CHECK - if (app_buf_addr >= memory_inst->memory_data_size) { - goto fail; - } - - if (!is_str) { - if (app_buf_size > memory_inst->memory_data_size - app_buf_addr) { - goto fail; - } - } - else { - const char *str, *str_end; - - /* The whole string must be in the linear memory */ - str = (const char *)native_addr; - str_end = (const char *)memory_inst->memory_data_end.ptr; - while (str < str_end && *str != '\0') - str++; - if (str == str_end) - goto fail; - } -#endif - - *p_native_addr = (void *)native_addr; - return true; -fail: - aot_set_exception(module_inst, "out of bounds memory access"); - return false; + return wasm_check_app_addr_and_convert(module_inst, is_str, app_buf_addr, + app_buf_size, p_native_addr); } void * @@ -2528,20 +1971,12 @@ aot_memory_init(AOTModuleInstance *module_inst, uint32 seg_index, uint32 offset, uint8 *maddr; uint64 seg_len = 0; - aot_module = (AOTModule *)module_inst->aot_module.ptr; - if (aot_module->is_jit_mode) { -#if WASM_ENABLE_JIT != 0 - seg_len = - aot_module->wasm_module->data_segments[seg_index]->data_length; - data = aot_module->wasm_module->data_segments[seg_index]->data; -#endif - } - else { - seg_len = aot_module->mem_init_data_list[seg_index]->byte_count; - data = aot_module->mem_init_data_list[seg_index]->bytes; - } + aot_module = (AOTModule *)module_inst->module; + seg_len = aot_module->mem_init_data_list[seg_index]->byte_count; + data = aot_module->mem_init_data_list[seg_index]->bytes; - if (!aot_validate_app_addr(module_inst, dst, len)) + if (!wasm_runtime_validate_app_addr((WASMModuleInstanceCommon *)module_inst, + dst, len)) return false; if ((uint64)offset + (uint64)len > seg_len) { @@ -2549,7 +1984,8 @@ aot_memory_init(AOTModuleInstance *module_inst, uint32 seg_index, uint32 offset, return false; } - maddr = aot_addr_app_to_native(module_inst, dst); + maddr = wasm_runtime_addr_app_to_native( + (WASMModuleInstanceCommon *)module_inst, dst); bh_memcpy_s(maddr, memory_inst->memory_data_size - dst, data + offset, len); return true; @@ -2558,20 +1994,11 @@ aot_memory_init(AOTModuleInstance *module_inst, uint32 seg_index, uint32 offset, bool aot_data_drop(AOTModuleInstance *module_inst, uint32 seg_index) { - AOTModule *aot_module = (AOTModule *)(module_inst->aot_module.ptr); + AOTModule *aot_module = (AOTModule *)module_inst->module; - if (aot_module->is_jit_mode) { -#if WASM_ENABLE_JIT != 0 - aot_module->wasm_module->data_segments[seg_index]->data_length = 0; - /* Currently we can't free the dropped data segment - as they are stored in wasm bytecode */ -#endif - } - else { - aot_module->mem_init_data_list[seg_index]->byte_count = 0; - /* Currently we can't free the dropped data segment - as the mem_init_data_count is a continuous array */ - } + aot_module->mem_init_data_list[seg_index]->byte_count = 0; + /* Currently we can't free the dropped data segment + as the mem_init_data_count is a continuous array */ return true; } #endif /* WASM_ENABLE_BULK_MEMORY */ @@ -2581,7 +2008,7 @@ bool aot_set_aux_stack(WASMExecEnv *exec_env, uint32 start_offset, uint32 size) { AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst; - AOTModule *module = (AOTModule *)module_inst->aot_module.ptr; + AOTModule *module = (AOTModule *)module_inst->module; uint32 stack_top_idx = module->aux_stack_top_global_index; uint32 data_end = module->aux_data_end; @@ -2597,8 +2024,7 @@ aot_set_aux_stack(WASMExecEnv *exec_env, uint32 start_offset, uint32 size) /* The aux stack top is a wasm global, set the initial value for the global */ uint32 global_offset = module->globals[stack_top_idx].data_offset; - uint8 *global_addr = - (uint8 *)module_inst->global_data.ptr + global_offset; + uint8 *global_addr = module_inst->global_data + global_offset; *(int32 *)global_addr = start_offset; /* The aux stack boundary is a constant value, @@ -2615,7 +2041,7 @@ bool aot_get_aux_stack(WASMExecEnv *exec_env, uint32 *start_offset, uint32 *size) { AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst; - AOTModule *module = (AOTModule *)module_inst->aot_module.ptr; + AOTModule *module = (AOTModule *)module_inst->module; /* The aux stack information is resolved in loader and store in module */ @@ -2738,31 +2164,31 @@ aot_get_module_inst_mem_consumption(const AOTModuleInstance *module_inst, mem_conspn->module_inst_struct_size = sizeof(AOTModuleInstance); mem_conspn->memories_size = - sizeof(AOTPointer) * module_inst->memory_count + sizeof(void *) * module_inst->memory_count + sizeof(AOTMemoryInstance) * module_inst->memory_count; for (i = 0; i < module_inst->memory_count; i++) { - AOTMemoryInstance *mem_inst = - ((AOTMemoryInstance **)module_inst->memories.ptr)[i]; + AOTMemoryInstance *mem_inst = module_inst->memories[i]; mem_conspn->memories_size += mem_inst->num_bytes_per_page * mem_inst->cur_page_count; - mem_conspn->app_heap_size = (uint8 *)mem_inst->heap_data_end.ptr - - (uint8 *)mem_inst->heap_data.ptr; + mem_conspn->app_heap_size = + mem_inst->heap_data_end - mem_inst->heap_data; /* size of app heap structure */ mem_conspn->memories_size += mem_allocator_get_heap_struct_size(); } - tbl_inst = module_inst->tables.ptr; + mem_conspn->tables_size += + sizeof(AOTTableInstance *) * module_inst->table_count; for (i = 0; i < module_inst->table_count; i++) { - mem_conspn->tables_size += offsetof(AOTTableInstance, data); + tbl_inst = module_inst->tables[i]; + mem_conspn->tables_size += offsetof(AOTTableInstance, elems); mem_conspn->tables_size += sizeof(uint32) * tbl_inst->max_size; - tbl_inst = aot_next_tbl_inst(tbl_inst); } /* func_ptrs and func_type_indexes */ mem_conspn->functions_size = (sizeof(void *) + sizeof(uint32)) - * (((AOTModule *)module_inst->aot_module.ptr)->import_func_count - + ((AOTModule *)module_inst->aot_module.ptr)->func_count); + * (((AOTModule *)module_inst->module)->import_func_count + + ((AOTModule *)module_inst->module)->func_count); mem_conspn->globals_size = module_inst->global_data_size; @@ -2783,7 +2209,7 @@ aot_get_module_inst_mem_consumption(const AOTModuleInstance *module_inst, void aot_drop_table_seg(AOTModuleInstance *module_inst, uint32 tbl_seg_idx) { - AOTModule *module = (AOTModule *)module_inst->aot_module.ptr; + AOTModule *module = (AOTModule *)module_inst->module; AOTTableInitData *tbl_seg = module->table_init_data_list[tbl_seg_idx]; tbl_seg->is_dropped = true; } @@ -2795,9 +2221,9 @@ aot_table_init(AOTModuleInstance *module_inst, uint32 tbl_idx, { AOTTableInstance *tbl_inst; AOTTableInitData *tbl_seg; - const AOTModule *module = module_inst->aot_module.ptr; + const AOTModule *module = (AOTModule *)module_inst->module; - tbl_inst = aot_get_table_inst(module_inst, tbl_idx); + tbl_inst = module_inst->tables[tbl_idx]; bh_assert(tbl_inst); tbl_seg = module->table_init_data_list[tbl_seg_idx]; @@ -2823,7 +2249,7 @@ aot_table_init(AOTModuleInstance *module_inst, uint32 tbl_idx, return; } - bh_memcpy_s((uint8 *)tbl_inst + offsetof(AOTTableInstance, data) + bh_memcpy_s((uint8 *)tbl_inst + offsetof(AOTTableInstance, elems) + dst_offset * sizeof(uint32), (tbl_inst->cur_size - dst_offset) * sizeof(uint32), tbl_seg->func_indexes + src_offset, length * sizeof(uint32)); @@ -2836,10 +2262,10 @@ aot_table_copy(AOTModuleInstance *module_inst, uint32 src_tbl_idx, { AOTTableInstance *src_tbl_inst, *dst_tbl_inst; - src_tbl_inst = aot_get_table_inst(module_inst, src_tbl_idx); + src_tbl_inst = module_inst->tables[src_tbl_idx]; bh_assert(src_tbl_inst); - dst_tbl_inst = aot_get_table_inst(module_inst, dst_tbl_idx); + dst_tbl_inst = module_inst->tables[dst_tbl_idx]; bh_assert(dst_tbl_inst); if ((uint64)dst_offset + length > dst_tbl_inst->cur_size @@ -2851,10 +2277,10 @@ aot_table_copy(AOTModuleInstance *module_inst, uint32 src_tbl_idx, /* if src_offset >= dst_offset, copy from front to back */ /* if src_offset < dst_offset, copy from back to front */ /* merge all together */ - bh_memmove_s((uint8 *)(dst_tbl_inst) + offsetof(AOTTableInstance, data) + bh_memmove_s((uint8 *)dst_tbl_inst + offsetof(AOTTableInstance, elems) + dst_offset * sizeof(uint32), (dst_tbl_inst->cur_size - dst_offset) * sizeof(uint32), - (uint8 *)(src_tbl_inst) + offsetof(AOTTableInstance, data) + (uint8 *)src_tbl_inst + offsetof(AOTTableInstance, elems) + src_offset * sizeof(uint32), length * sizeof(uint32)); } @@ -2865,7 +2291,7 @@ aot_table_fill(AOTModuleInstance *module_inst, uint32 tbl_idx, uint32 length, { AOTTableInstance *tbl_inst; - tbl_inst = aot_get_table_inst(module_inst, tbl_idx); + tbl_inst = module_inst->tables[tbl_idx]; bh_assert(tbl_inst); if (data_offset + length > tbl_inst->cur_size) { @@ -2874,7 +2300,7 @@ aot_table_fill(AOTModuleInstance *module_inst, uint32 tbl_idx, uint32 length, } for (; length != 0; data_offset++, length--) { - tbl_inst->data[data_offset] = val; + tbl_inst->elems[data_offset] = val; } } @@ -2885,7 +2311,7 @@ aot_table_grow(AOTModuleInstance *module_inst, uint32 tbl_idx, uint32 entry_count, i, orig_tbl_sz; AOTTableInstance *tbl_inst; - tbl_inst = aot_get_table_inst(module_inst, tbl_idx); + tbl_inst = module_inst->tables[tbl_idx]; if (!tbl_inst) { return (uint32)-1; } @@ -2907,7 +2333,7 @@ aot_table_grow(AOTModuleInstance *module_inst, uint32 tbl_idx, /* fill in */ for (i = 0; i < inc_entries; ++i) { - tbl_inst->data[tbl_inst->cur_size + i] = init_val; + tbl_inst->elems[tbl_inst->cur_size + i] = init_val; } tbl_inst->cur_size = entry_count; @@ -2947,7 +2373,7 @@ get_func_name_from_index(const AOTModuleInstance *module_inst, uint32 func_index) { const char *func_name = NULL; - AOTModule *module = module_inst->aot_module.ptr; + AOTModule *module = (AOTModule *)module_inst->module; #if WASM_ENABLE_CUSTOM_NAME_SECTION != 0 if ((func_name = @@ -2983,8 +2409,7 @@ aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index) #if WASM_ENABLE_PERF_PROFILING != 0 AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst; AOTFuncPerfProfInfo *func_perf_prof = - (AOTFuncPerfProfInfo *)module_inst->func_perf_profilings.ptr - + func_index; + module_inst->func_perf_profilings + func_index; #endif if (!frame) { @@ -3038,8 +2463,8 @@ aot_create_call_stack(struct WASMExecEnv *exec_env) } /* release previous stack frames and create new ones */ - if (!bh_vector_destroy(module_inst->frames.ptr) - || !bh_vector_init(module_inst->frames.ptr, n, sizeof(WASMCApiFrame), + if (!bh_vector_destroy(module_inst->frames) + || !bh_vector_init(module_inst->frames, n, sizeof(WASMCApiFrame), false)) { return false; } @@ -3054,8 +2479,8 @@ aot_create_call_stack(struct WASMExecEnv *exec_env) frame.func_name_wp = get_func_name_from_index(module_inst, cur_frame->func_index); - if (!bh_vector_append(module_inst->frames.ptr, &frame)) { - bh_vector_destroy(module_inst->frames.ptr); + if (!bh_vector_append(module_inst->frames, &frame)) { + bh_vector_destroy(module_inst->frames); return false; } @@ -3083,11 +2508,11 @@ aot_dump_call_stack(WASMExecEnv *exec_env, bool print, char *buf, uint32 len) * will be truncated */ char line_buf[256]; - if (!module_inst->frames.ptr) { + if (!module_inst->frames) { return 0; } - total_frames = (uint32)bh_vector_size(module_inst->frames.ptr); + total_frames = (uint32)bh_vector_size(module_inst->frames); if (total_frames == 0) { return 0; } @@ -3099,7 +2524,7 @@ aot_dump_call_stack(WASMExecEnv *exec_env, bool print, char *buf, uint32 len) WASMCApiFrame frame = { 0 }; uint32 line_length, i; - if (!bh_vector_get(module_inst->frames.ptr, n, &frame)) { + if (!bh_vector_get(module_inst->frames, n, &frame)) { return 0; } @@ -3138,8 +2563,8 @@ void aot_dump_perf_profiling(const AOTModuleInstance *module_inst) { AOTFuncPerfProfInfo *perf_prof = - (AOTFuncPerfProfInfo *)module_inst->func_perf_profilings.ptr; - AOTModule *module = (AOTModule *)module_inst->aot_module.ptr; + (AOTFuncPerfProfInfo *)module_inst->func_perf_profilings; + AOTModule *module = (AOTModule *)module_inst->module; uint32 total_func_count = module->import_func_count + module->func_count, i; const char *func_name; diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index 14e037e7..e5503ed1 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -10,34 +10,11 @@ #include "../common/wasm_runtime_common.h" #include "../interpreter/wasm_runtime.h" #include "../compilation/aot.h" -#if WASM_ENABLE_JIT != 0 -#include "../compilation/aot_llvm.h" -#endif #ifdef __cplusplus extern "C" { #endif -typedef enum AOTExceptionID { - EXCE_UNREACHABLE = 0, - EXCE_OUT_OF_MEMORY, - EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, - EXCE_INTEGER_OVERFLOW, - EXCE_INTEGER_DIVIDE_BY_ZERO, - EXCE_INVALID_CONVERSION_TO_INTEGER, - EXCE_INVALID_FUNCTION_TYPE_INDEX, - EXCE_INVALID_FUNCTION_INDEX, - EXCE_UNDEFINED_ELEMENT, - EXCE_UNINITIALIZED_ELEMENT, - EXCE_CALL_UNLINKED_IMPORT_FUNC, - EXCE_NATIVE_STACK_OVERFLOW, - EXCE_UNALIGNED_ATOMIC, - EXCE_AUX_STACK_OVERFLOW, - EXCE_AUX_STACK_UNDERFLOW, - EXCE_OUT_OF_BOUNDS_TABLE_ACCESS, - EXCE_NUM, -} AOTExceptionID; - typedef enum AOTSectionType { AOT_SECTION_TYPE_TARGET_INFO = 0, AOT_SECTION_TYPE_INIT_DATA = 1, @@ -171,7 +148,7 @@ typedef struct AOTModule { /* function info */ uint32 func_count; - /* point to AOTed/JITed functions */ + /* point to AOTed functions */ void **func_ptrs; /* function type indexes */ uint32 *func_type_indexes; @@ -182,24 +159,24 @@ typedef struct AOTModule { /* start function index, -1 denotes no start function */ uint32 start_func_index; - /* start function, point to AOTed/JITed function */ + /* start function, point to AOTed function */ void *start_function; uint32 malloc_func_index; uint32 free_func_index; uint32 retain_func_index; - /* AOTed code, NULL for JIT mode */ + /* AOTed code */ void *code; uint32 code_size; - /* literal for AOTed code, NULL for JIT mode */ + /* literal for AOTed code */ uint8 *literal; uint32 literal_size; #if defined(BH_PLATFORM_WINDOWS) /* extra plt data area for __ymm, __xmm and __real constants - in Windows platform, NULL for JIT mode */ + in Windows platform */ uint8 *extra_plt_data; uint32 extra_plt_data_size; uint32 ymm_plt_count; @@ -217,7 +194,7 @@ typedef struct AOTModule { #endif /* data sections in AOT object file, including .data, .rodata - * and .rodata.cstN. NULL for JIT mode. */ + and .rodata.cstN. */ AOTObjectDataSection *data_sections; uint32 data_section_count; @@ -244,18 +221,9 @@ typedef struct AOTModule { /* auxiliary stack size resolved */ uint32 aux_stack_size; - /* is jit mode or not */ - bool is_jit_mode; - /* is indirect mode or not */ bool is_indirect_mode; -#if WASM_ENABLE_JIT != 0 - WASMModule *wasm_module; - AOTCompContext *comp_ctx; - AOTCompData *comp_data; -#endif - #if WASM_ENABLE_LIBC_WASI != 0 WASIArguments wasi_args; bool import_wasi_api; @@ -274,130 +242,9 @@ typedef struct AOTModule { #endif } AOTModule; -typedef union { - uint64 _make_it_8_bytes_; - void *ptr; -} AOTPointer; - -typedef union { - uint64 u64; - uint32 u32[2]; -} MemBound; - -typedef struct AOTMemoryInstance { - uint32 module_type; - /* shared memory flag */ - bool is_shared; - - /* memory space info */ - uint32 num_bytes_per_page; - uint32 cur_page_count; - uint32 max_page_count; - uint32 memory_data_size; - AOTPointer memory_data; - AOTPointer memory_data_end; - - /* heap space info */ - AOTPointer heap_data; - AOTPointer heap_data_end; - AOTPointer heap_handle; - - /* boundary check constants for aot code */ - MemBound mem_bound_check_1byte; - MemBound mem_bound_check_2bytes; - MemBound mem_bound_check_4bytes; - MemBound mem_bound_check_8bytes; - MemBound mem_bound_check_16bytes; -} AOTMemoryInstance; - -typedef struct AOTTableInstance { - uint32 cur_size; - /* - * only grow in the range of [:max_size) - * if the table is growable, max_size equals to its declared maximum size - * otherwise, max_size equals to its declared minimum size - */ - uint32 max_size; - /* - * +------------------------------+ <--- data - * | ref.func[] or ref.extern[] - * +------------------------------+ - */ - uint32 data[1]; -} AOTTableInstance; - -typedef struct AOTModuleInstance { - uint32 module_type; - - /* memories */ - uint32 memory_count; - AOTPointer memories; - - /* global and table info */ - uint32 global_data_size; - /* - * the count of AOTTableInstance. - * it includes imported tables and local tables. - * - * TODO: for now we treate imported table like a local table - */ - uint32 table_count; - /* points to global_data */ - AOTPointer global_data; - /* points to AOTTableInstance[] */ - AOTPointer tables; - - /* function pointer array */ - AOTPointer func_ptrs; - /* function type indexes */ - AOTPointer func_type_indexes; - - /* export info */ - uint32 export_func_count; - uint32 export_global_count; - uint32 export_mem_count; - uint32 export_tab_count; - AOTPointer export_funcs; - AOTPointer export_globals; - AOTPointer export_memories; - AOTPointer export_tables; - - /* The exception buffer for current thread. */ - char cur_exception[128]; - /* The custom data that can be set/get by - * wasm_runtime_set_custom_data/wasm_runtime_get_custom_data */ - AOTPointer custom_data; - /* The AOT module */ - AOTPointer aot_module; - /* WASI context */ - AOTPointer wasi_ctx; - /* function performance profiling info list */ - AOTPointer func_perf_profilings; - /* stack frames, used in call stack dump and perf profiling */ - AOTPointer frames; - - AOTPointer exec_env_singleton; - - uint32 default_wasm_stack_size; - - /* reserved */ - uint32 reserved[9]; - - /* - * +------------------------------+ <-- memories.ptr - * | #0 AOTMemoryInstance - * +------------------------------+ <-- global_data.ptr - * | global data - * +------------------------------+ <-- tables.ptr - * | AOTTableInstance[table_count] - * +------------------------------+ - */ - union { - uint64 _make_it_8_byte_aligned_; - AOTMemoryInstance memory_instances[1]; - uint8 bytes[1]; - } global_table_data; -} AOTModuleInstance; +#define AOTMemoryInstance WASMMemoryInstance +#define AOTTableInstance WASMTableInstance +#define AOTModuleInstance WASMModuleInstance /* Target info, read from ELF header of object file */ typedef struct AOTTargetInfo { @@ -462,21 +309,6 @@ AOTModule * aot_load_from_sections(AOTSection *section_list, char *error_buf, uint32 error_buf_size); -#if WASM_ENABLE_JIT != 0 -/** - * Convert WASM module to AOT module - * - * @param wasm_module the WASM module to convert - * @param error_buf output of the error info - * @param error_buf_size the size of the error string - * - * @return return AOT module loaded, NULL if failed - */ -AOTModule * -aot_convert_wasm_module(WASMModule *wasm_module, char *error_buf, - uint32 error_buf_size); -#endif - /** * Unload a AOT module. * @@ -549,9 +381,6 @@ aot_create_exec_env_and_call_function(AOTModuleInstance *module_inst, AOTFunctionInstance *function, unsigned argc, uint32 argv[]); -bool -aot_create_exec_env_singleton(AOTModuleInstance *module_inst); - /** * Set AOT module instance exception with exception string * @@ -575,14 +404,6 @@ aot_set_exception_with_id(AOTModuleInstance *module_inst, uint32 id); const char * aot_get_exception(AOTModuleInstance *module_inst); -/** - * Clear exception info of the AOT module instance. - * - * @param module_inst the AOT module instance - */ -void -aot_clear_exception(AOTModuleInstance *module_inst); - uint32 aot_module_malloc(AOTModuleInstance *module_inst, uint32 size, void **p_native_addr); @@ -598,29 +419,6 @@ uint32 aot_module_dup_data(AOTModuleInstance *module_inst, const char *src, uint32 size); -bool -aot_validate_app_addr(AOTModuleInstance *module_inst, uint32 app_offset, - uint32 size); - -bool -aot_validate_native_addr(AOTModuleInstance *module_inst, void *native_ptr, - uint32 size); - -void * -aot_addr_app_to_native(AOTModuleInstance *module_inst, uint32 app_offset); - -uint32 -aot_addr_native_to_app(AOTModuleInstance *module_inst, void *native_ptr); - -bool -aot_get_app_addr_range(AOTModuleInstance *module_inst, uint32 app_offset, - uint32 *p_app_start_offset, uint32 *p_app_end_offset); - -bool -aot_get_native_addr_range(AOTModuleInstance *module_inst, uint8 *native_ptr, - uint8 **p_native_start_addr, - uint8 **p_native_end_addr); - bool aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count); @@ -635,6 +433,10 @@ bool aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, uint32 argc, uint32 *argv); +/** + * Check whether the app address and the buf is inside the linear memory, + * and convert the app address into native address + */ bool aot_check_app_addr_and_convert(AOTModuleInstance *module_inst, bool is_str, uint32 app_buf_addr, uint32 app_buf_size, @@ -666,16 +468,6 @@ bool aot_get_aux_stack(WASMExecEnv *exec_env, uint32 *start_offset, uint32 *size); #endif -#ifdef OS_ENABLE_HW_BOUND_CHECK -#ifndef BH_PLATFORM_WINDOWS -void -aot_signal_handler(WASMSignalInfo *sig_info); -#else -LONG -aot_exception_handler(WASMSignalInfo *sig_info); -#endif -#endif - void aot_get_module_mem_consumption(const AOTModule *module, WASMModuleMemConsumption *mem_conspn); @@ -707,9 +499,6 @@ aot_table_grow(AOTModuleInstance *module_inst, uint32 tbl_idx, uint32 inc_entries, uint32 init_val); #endif -AOTTableInstance * -aot_next_tbl_inst(const AOTTableInstance *tbl_inst); - bool aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index); diff --git a/core/iwasm/common/wasm_c_api.c b/core/iwasm/common/wasm_c_api.c index 52b6464a..4b1958fe 100644 --- a/core/iwasm/common/wasm_c_api.c +++ b/core/iwasm/common/wasm_c_api.c @@ -2,16 +2,20 @@ * Copyright (C) 2019 Intel Corporation. All rights reserved. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ - #include "wasm_c_api_internal.h" + +#include "bh_assert.h" #include "wasm_memory.h" -#include "wasm_runtime_common.h" #if WASM_ENABLE_INTERP != 0 #include "wasm_runtime.h" #endif #if WASM_ENABLE_AOT != 0 #include "aot_runtime.h" -#endif +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT == 0 +#include "aot.h" +#include "aot_llvm.h" +#endif /*WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT == 0*/ +#endif /*WASM_ENABLE_AOT != 0*/ #define ASSERT_NOT_IMPLEMENTED() bh_assert(!"not implemented") #define UNREACHABLE() bh_assert(!"unreachable") @@ -302,9 +306,11 @@ wasm_engine_new_internal(mem_alloc_type_t type, const MemAllocOption *opts) goto failed; } -#if BH_DEBUG != 0 +#ifndef NDEBUG + /*DEBUG*/ bh_log_set_verbose_level(5); #else + /*VERBOSE*/ bh_log_set_verbose_level(3); #endif @@ -1655,7 +1661,7 @@ wasm_trap_new_internal(WASMModuleInstanceCommon *inst_comm_rt, #if WASM_ENABLE_AOT != 0 if (inst_comm_rt->module_type == Wasm_Module_AoT) { - trap->frames = ((AOTModuleInstance *)inst_comm_rt)->frames.ptr; + trap->frames = ((AOTModuleInstance *)inst_comm_rt)->frames; } #endif #endif /* WASM_ENABLE_DUMP_CALL_STACK != 0 */ @@ -2379,6 +2385,61 @@ failed_exporttype_new: wasm_exporttype_vec_delete(out); } +#if WASM_ENABLE_JIT == 0 || WASM_ENABLE_LAZY_JIT != 0 +void +wasm_module_serialize(wasm_module_t *module, own wasm_byte_vec_t *out) +{ + (void)module; + (void)out; + LOG_ERROR("only supported serialization in JIT with eager compilation"); +} + +own wasm_module_t * +wasm_module_deserialize(wasm_store_t *module, const wasm_byte_vec_t *binary) +{ + (void)module; + (void)binary; + LOG_ERROR("only supported deserialization in JIT with eager compilation"); + return NULL; +} +#else + +extern uint8 * +aot_emit_aot_file_buf(AOTCompContext *comp_ctx, AOTCompData *comp_data, + uint32 *p_aot_file_size); +void +wasm_module_serialize(wasm_module_t *module, own wasm_byte_vec_t *out) +{ + wasm_module_ex_t *module_ex; + AOTCompContext *comp_ctx; + AOTCompData *comp_data; + uint8 *aot_file_buf = NULL; + uint32 aot_file_size = 0; + + if (!module || !out) + return; + + module_ex = module_to_module_ext(module); + comp_ctx = ((WASMModule *)(module_ex->module_comm_rt))->comp_ctx; + comp_data = ((WASMModule *)(module_ex->module_comm_rt))->comp_data; + bh_assert(comp_ctx != NULL && comp_data != NULL); + + aot_file_buf = aot_emit_aot_file_buf(comp_ctx, comp_data, &aot_file_size); + if (!aot_file_buf) + return; + + wasm_byte_vec_new(out, aot_file_size, (wasm_byte_t *)aot_file_buf); + wasm_runtime_free(aot_file_buf); + return; +} + +own wasm_module_t * +wasm_module_deserialize(wasm_store_t *store, const wasm_byte_vec_t *binary) +{ + return wasm_module_new(store, binary); +} +#endif + static wasm_func_t * wasm_func_new_basic(wasm_store_t *store, const wasm_functype_t *type, wasm_func_callback_t func_callback) @@ -2482,9 +2543,9 @@ wasm_func_new_internal(wasm_store_t *store, uint16 func_idx_rt, #if WASM_ENABLE_INTERP != 0 if (inst_comm_rt->module_type == Wasm_Module_Bytecode) { bh_assert(func_idx_rt - < ((WASMModuleInstance *)inst_comm_rt)->function_count); + < ((WASMModuleInstance *)inst_comm_rt)->e->function_count); WASMFunctionInstance *func_interp = - ((WASMModuleInstance *)inst_comm_rt)->functions + func_idx_rt; + ((WASMModuleInstance *)inst_comm_rt)->e->functions + func_idx_rt; type_rt = func_interp->is_import_func ? func_interp->u.func_import->func_type : func_interp->u.func->func_type; @@ -2496,7 +2557,7 @@ wasm_func_new_internal(wasm_store_t *store, uint16 func_idx_rt, /* use same index to trace the function type in AOTFuncType **func_types */ AOTModule *module_aot = - ((AOTModuleInstance *)inst_comm_rt)->aot_module.ptr; + (AOTModule *)((AOTModuleInstance *)inst_comm_rt)->module; if (func_idx_rt < module_aot->import_func_count) { type_rt = (module_aot->import_funcs + func_idx_rt)->func_type; } @@ -2605,6 +2666,7 @@ params_to_argv(const wasm_val_vec_t *params, } if (!params || !params->num_elems || !params->size || !params->data) { + LOG_ERROR("the parameter params is invalid"); return false; } @@ -2662,6 +2724,7 @@ argv_to_results(const uint32 *argv, const wasm_valtype_vec_t *result_defs, } if (!results || !results->size || !results->data) { + LOG_ERROR("the parameter results is invalid"); return false; } @@ -2748,7 +2811,7 @@ wasm_func_call(const wasm_func_t *func, const wasm_val_vec_t *params, #if WASM_ENABLE_INTERP != 0 if (func->inst_comm_rt->module_type == Wasm_Module_Bytecode) { - func_comm_rt = ((WASMModuleInstance *)func->inst_comm_rt)->functions + func_comm_rt = ((WASMModuleInstance *)func->inst_comm_rt)->e->functions + func->func_idx_rt; } #endif @@ -2758,7 +2821,7 @@ wasm_func_call(const wasm_func_t *func, const wasm_val_vec_t *params, if (!(func_comm_rt = func->func_comm_rt)) { AOTModuleInstance *inst_aot = (AOTModuleInstance *)func->inst_comm_rt; - AOTModule *module_aot = (AOTModule *)inst_aot->aot_module.ptr; + AOTModule *module_aot = (AOTModule *)inst_aot->module; uint32 export_i = 0, export_func_j = 0; for (; export_i < module_aot->export_count; ++export_i) { @@ -2766,7 +2829,7 @@ wasm_func_call(const wasm_func_t *func, const wasm_val_vec_t *params, if (export->kind == EXPORT_KIND_FUNC) { if (export->index == func->func_idx_rt) { func_comm_rt = - (AOTFunctionInstance *)inst_aot->export_funcs.ptr + (AOTFunctionInstance *)inst_aot->export_functions + export_func_j; ((wasm_func_t *)func)->func_comm_rt = func_comm_rt; break; @@ -2788,6 +2851,7 @@ wasm_func_call(const wasm_func_t *func, const wasm_val_vec_t *params, param_count = wasm_func_param_arity(func); result_count = wasm_func_result_arity(func); + alloc_count = (param_count > result_count) ? param_count : result_count; if (alloc_count > (size_t)sizeof(argv_buf) / sizeof(uint64)) { if (!(argv = malloc_internal(sizeof(uint64) * alloc_count))) { @@ -2968,7 +3032,7 @@ interp_global_set(const WASMModuleInstance *inst_interp, uint16 global_idx_rt, const wasm_val_t *v) { const WASMGlobalInstance *global_interp = - inst_interp->globals + global_idx_rt; + inst_interp->e->globals + global_idx_rt; uint8 val_type_rt = global_interp->type; #if WASM_ENABLE_MULTI_MODULE != 0 uint8 *data = global_interp->import_global_inst @@ -2987,7 +3051,7 @@ static bool interp_global_get(const WASMModuleInstance *inst_interp, uint16 global_idx_rt, wasm_val_t *out) { - WASMGlobalInstance *global_interp = inst_interp->globals + global_idx_rt; + WASMGlobalInstance *global_interp = inst_interp->e->globals + global_idx_rt; uint8 val_type_rt = global_interp->type; #if WASM_ENABLE_MULTI_MODULE != 0 uint8 *data = global_interp->import_global_inst @@ -3007,7 +3071,7 @@ static bool aot_global_set(const AOTModuleInstance *inst_aot, uint16 global_idx_rt, const wasm_val_t *v) { - AOTModule *module_aot = inst_aot->aot_module.ptr; + AOTModule *module_aot = (AOTModule *)inst_aot->module; uint8 val_type_rt = 0; uint32 data_offset = 0; void *data = NULL; @@ -3025,7 +3089,7 @@ aot_global_set(const AOTModuleInstance *inst_aot, uint16 global_idx_rt, .type; } - data = (void *)((uint8 *)inst_aot->global_data.ptr + data_offset); + data = (void *)(inst_aot->global_data + data_offset); return wasm_val_to_rt_val((WASMModuleInstanceCommon *)inst_aot, val_type_rt, v, data); } @@ -3034,7 +3098,7 @@ static bool aot_global_get(const AOTModuleInstance *inst_aot, uint16 global_idx_rt, wasm_val_t *out) { - AOTModule *module_aot = inst_aot->aot_module.ptr; + AOTModule *module_aot = (AOTModule *)inst_aot->module; uint8 val_type_rt = 0; uint32 data_offset = 0; uint8 *data = NULL; @@ -3052,7 +3116,7 @@ aot_global_get(const AOTModuleInstance *inst_aot, uint16 global_idx_rt, .type; } - data = (uint8 *)inst_aot->global_data.ptr + data_offset; + data = inst_aot->global_data + data_offset; return rt_val_to_wasm_val(data, val_type_rt, out); } #endif @@ -3149,7 +3213,7 @@ wasm_global_new_internal(wasm_store_t *store, uint16 global_idx_rt, #if WASM_ENABLE_INTERP != 0 if (inst_comm_rt->module_type == Wasm_Module_Bytecode) { WASMGlobalInstance *global_interp = - ((WASMModuleInstance *)inst_comm_rt)->globals + global_idx_rt; + ((WASMModuleInstance *)inst_comm_rt)->e->globals + global_idx_rt; val_type_rt = global_interp->type; is_mutable = global_interp->is_mutable; init = true; @@ -3159,7 +3223,7 @@ wasm_global_new_internal(wasm_store_t *store, uint16 global_idx_rt, #if WASM_ENABLE_AOT != 0 if (inst_comm_rt->module_type == Wasm_Module_AoT) { AOTModuleInstance *inst_aot = (AOTModuleInstance *)inst_comm_rt; - AOTModule *module_aot = inst_aot->aot_module.ptr; + AOTModule *module_aot = (AOTModule *)inst_aot->module; init = true; @@ -3257,7 +3321,6 @@ wasm_table_new_internal(wasm_store_t *store, uint16 table_idx_rt, wasm_table_t *table = NULL; uint8 val_type_rt = 0; uint32 init_size = 0, max_size = 0; - bool init_flag = false; bh_assert(singleton_engine); @@ -3272,46 +3335,12 @@ wasm_table_new_internal(wasm_store_t *store, uint16 table_idx_rt, table->store = store; table->kind = WASM_EXTERN_TABLE; -#if WASM_ENABLE_INTERP != 0 - if (inst_comm_rt->module_type == Wasm_Module_Bytecode) { - WASMTableInstance *table_interp = - ((WASMModuleInstance *)inst_comm_rt)->tables[table_idx_rt]; - val_type_rt = table_interp->elem_type; - init_size = table_interp->cur_size; - max_size = table_interp->max_size; - init_flag = true; - } -#endif - -#if WASM_ENABLE_AOT != 0 - if (inst_comm_rt->module_type == Wasm_Module_AoT) { - AOTModuleInstance *inst_aot = (AOTModuleInstance *)inst_comm_rt; - AOTModule *module_aot = (AOTModule *)inst_aot->aot_module.ptr; - - if (table_idx_rt < module_aot->import_table_count) { - AOTImportTable *table_aot = - module_aot->import_tables + table_idx_rt; - val_type_rt = table_aot->elem_type; - init_size = table_aot->table_init_size; - max_size = table_aot->table_max_size; - } - else { - AOTTable *table_aot = - module_aot->tables - + (table_idx_rt - module_aot->import_table_count); - val_type_rt = table_aot->elem_type; - init_size = table_aot->table_init_size; - max_size = table_aot->table_max_size; - } - init_flag = true; - } -#endif - - /* - * a wrong combination of module filetype and compilation flags - * leads to below branch - */ - if (!init_flag) { + if (!wasm_runtime_get_table_inst_elem_type( + inst_comm_rt, table_idx_rt, &val_type_rt, &init_size, &max_size)) { + /* + * a wrong combination of module filetype and compilation flags + * leads to below branch + */ goto failed; } @@ -3400,19 +3429,18 @@ wasm_table_get(const wasm_table_t *table, wasm_table_size_t index) if (index >= table_interp->cur_size) { return NULL; } - ref_idx = ((uint32 *)table_interp->base_addr)[index]; + ref_idx = table_interp->elems[index]; } #endif #if WASM_ENABLE_AOT != 0 if (table->inst_comm_rt->module_type == Wasm_Module_AoT) { AOTModuleInstance *inst_aot = (AOTModuleInstance *)table->inst_comm_rt; - AOTTableInstance *table_aot = - (AOTTableInstance *)inst_aot->tables.ptr + table->table_idx_rt; + AOTTableInstance *table_aot = inst_aot->tables[table->table_idx_rt]; if (index >= table_aot->cur_size) { return NULL; } - ref_idx = table_aot->data[index]; + ref_idx = table_aot->elems[index]; } #endif @@ -3472,24 +3500,23 @@ wasm_table_set(wasm_table_t *table, wasm_table_size_t index, return false; } - p_ref_idx = ((uint32 *)table_interp->base_addr) + index; + p_ref_idx = table_interp->elems + index; function_count = - ((WASMModuleInstance *)table->inst_comm_rt)->function_count; + ((WASMModuleInstance *)table->inst_comm_rt)->e->function_count; } #endif #if WASM_ENABLE_AOT != 0 if (table->inst_comm_rt->module_type == Wasm_Module_AoT) { AOTModuleInstance *inst_aot = (AOTModuleInstance *)table->inst_comm_rt; - AOTModule *module_aot = (AOTModule *)inst_aot->aot_module.ptr; - AOTTableInstance *table_aot = - (AOTTableInstance *)inst_aot->tables.ptr + table->table_idx_rt; + AOTModule *module_aot = (AOTModule *)inst_aot->module; + AOTTableInstance *table_aot = inst_aot->tables[table->table_idx_rt]; if (index >= table_aot->cur_size) { return false; } - p_ref_idx = table_aot->data + index; + p_ref_idx = table_aot->elems + index; function_count = module_aot->func_count; } #endif @@ -3545,7 +3572,7 @@ wasm_table_size(const wasm_table_t *table) #if WASM_ENABLE_AOT != 0 if (table->inst_comm_rt->module_type == Wasm_Module_AoT) { AOTModuleInstance *inst_aot = (AOTModuleInstance *)table->inst_comm_rt; - AOTModule *module_aot = (AOTModule *)inst_aot->aot_module.ptr; + AOTModule *module_aot = (AOTModule *)inst_aot->module; if (table->table_idx_rt < module_aot->import_table_count) { AOTImportTable *table_aot = @@ -3660,7 +3687,7 @@ wasm_memory_new_internal(wasm_store_t *store, uint16 memory_idx_rt, #if WASM_ENABLE_AOT != 0 if (inst_comm_rt->module_type == Wasm_Module_AoT) { AOTModuleInstance *inst_aot = (AOTModuleInstance *)inst_comm_rt; - AOTModule *module_aot = (AOTModule *)(inst_aot->aot_module.ptr); + AOTModule *module_aot = (AOTModule *)inst_aot->module; if (memory_idx_rt < module_aot->import_memory_count) { min_pages = module_aot->import_memories->mem_init_page_count; @@ -3744,8 +3771,8 @@ wasm_memory_data(wasm_memory_t *memory) AOTModuleInstance *module_inst = (AOTModuleInstance *)module_inst_comm; AOTMemoryInstance *memory_inst = ((AOTMemoryInstance **) - module_inst->memories.ptr)[memory->memory_idx_rt]; - return (byte_t *)memory_inst->memory_data.ptr; + module_inst->memories)[memory->memory_idx_rt]; + return (byte_t *)memory_inst->memory_data; } #endif @@ -3781,7 +3808,7 @@ wasm_memory_data_size(const wasm_memory_t *memory) AOTModuleInstance *module_inst = (AOTModuleInstance *)module_inst_comm; AOTMemoryInstance *memory_inst = ((AOTMemoryInstance **) - module_inst->memories.ptr)[memory->memory_idx_rt]; + module_inst->memories)[memory->memory_idx_rt]; return memory_inst->cur_page_count * memory_inst->num_bytes_per_page; } #endif @@ -3818,7 +3845,7 @@ wasm_memory_size(const wasm_memory_t *memory) AOTModuleInstance *module_inst = (AOTModuleInstance *)module_inst_comm; AOTMemoryInstance *memory_inst = ((AOTMemoryInstance **) - module_inst->memories.ptr)[memory->memory_idx_rt]; + module_inst->memories)[memory->memory_idx_rt]; return memory_inst->cur_page_count; } #endif @@ -4197,7 +4224,7 @@ aot_process_export(wasm_store_t *store, const AOTModuleInstance *inst_aot, bh_assert(store && inst_aot && externals); - module_aot = (AOTModule *)inst_aot->aot_module.ptr; + module_aot = (AOTModule *)inst_aot->module; bh_assert(module_aot); for (i = 0; i < module_aot->export_count; ++i) { @@ -4420,8 +4447,9 @@ wasm_instance_new_with_args(wasm_store_t *store, const wasm_module_t *module, uint32 export_cnt = ((AOTModuleInstance *)instance->inst_comm_rt)->export_func_count + ((AOTModuleInstance *)instance->inst_comm_rt)->export_global_count - + ((AOTModuleInstance *)instance->inst_comm_rt)->export_tab_count - + ((AOTModuleInstance *)instance->inst_comm_rt)->export_mem_count; + + ((AOTModuleInstance *)instance->inst_comm_rt)->export_table_count + + ((AOTModuleInstance *)instance->inst_comm_rt) + ->export_memory_count; INIT_VEC(instance->exports, wasm_extern_vec_new_uninitialized, export_cnt); diff --git a/core/iwasm/common/wasm_exec_env.c b/core/iwasm/common/wasm_exec_env.c index ae05cce7..cd8b5708 100644 --- a/core/iwasm/common/wasm_exec_env.c +++ b/core/iwasm/common/wasm_exec_env.c @@ -65,7 +65,7 @@ wasm_exec_env_create_internal(struct WASMModuleInstanceCommon *module_inst, #if WASM_ENABLE_AOT != 0 if (module_inst->module_type == Wasm_Module_AoT) { AOTModuleInstance *i = (AOTModuleInstance *)module_inst; - AOTModule *m = (AOTModule *)i->aot_module.ptr; + AOTModule *m = (AOTModule *)i->module; exec_env->native_symbol = m->native_symbol_list; } #endif @@ -135,7 +135,7 @@ wasm_exec_env_create(struct WASMModuleInstanceCommon *module_inst, /* Set the aux_stack_boundary and aux_stack_bottom */ if (module_inst->module_type == Wasm_Module_AoT) { AOTModule *module = - (AOTModule *)(((AOTModuleInstance *)module_inst)->aot_module.ptr); + (AOTModule *)((AOTModuleInstance *)module_inst)->module; exec_env->aux_stack_bottom.bottom = module->aux_stack_bottom; exec_env->aux_stack_boundary.boundary = module->aux_stack_bottom - module->aux_stack_size; diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 49a9b086..ffe21857 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -4,6 +4,7 @@ */ #include "wasm_runtime_common.h" +#include "../interpreter/wasm_runtime.h" #include "bh_platform.h" #include "mem_alloc.h" @@ -138,6 +139,9 @@ wasm_runtime_free_internal(void *ptr) { if (!ptr) { LOG_WARNING("warning: wasm_runtime_free with NULL pointer\n"); +#if BH_ENABLE_GC_VERIFY != 0 + exit(-1); +#endif return; } @@ -160,6 +164,9 @@ wasm_runtime_malloc(unsigned int size) LOG_WARNING("warning: wasm_runtime_malloc with size zero\n"); /* At lease alloc 1 byte to avoid malloc failed */ size = 1; +#if BH_ENABLE_GC_VERIFY != 0 + exit(-1); +#endif } return wasm_runtime_malloc_internal(size); @@ -185,3 +192,446 @@ wasm_runtime_get_mem_alloc_info(mem_alloc_info_t *mem_alloc_info) } return false; } + +bool +wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, + uint32 app_offset, uint32 size) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + goto fail; + } + + /* integer overflow check */ + if (app_offset > UINT32_MAX - size) { + goto fail; + } + + if (app_offset + size <= memory_inst->memory_data_size) { + return true; + } + +fail: + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; +} + +bool +wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm, + uint32 app_str_offset) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + uint32 app_end_offset; + char *str, *str_end; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + if (!wasm_runtime_get_app_addr_range(module_inst_comm, app_str_offset, NULL, + &app_end_offset)) + goto fail; + + str = wasm_runtime_addr_app_to_native(module_inst_comm, app_str_offset); + str_end = str + (app_end_offset - app_str_offset); + while (str < str_end && *str != '\0') + str++; + if (str == str_end) + goto fail; + + return true; +fail: + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; +} + +bool +wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm, + void *native_ptr, uint32 size) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + uint8 *addr = (uint8 *)native_ptr; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + goto fail; + } + + /* integer overflow check */ + if ((uintptr_t)addr > UINTPTR_MAX - size) { + goto fail; + } + + if (memory_inst->memory_data <= addr + && addr + size <= memory_inst->memory_data_end) { + return true; + } + +fail: + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; +} + +void * +wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, + uint32 app_offset) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + uint8 *addr; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + return NULL; + } + + addr = memory_inst->memory_data + app_offset; + + if (memory_inst->memory_data <= addr && addr < memory_inst->memory_data_end) + return addr; + + return NULL; +} + +uint32 +wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, + void *native_ptr) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + uint8 *addr = (uint8 *)native_ptr; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + return 0; + } + + if (memory_inst->memory_data <= addr && addr < memory_inst->memory_data_end) + return (uint32)(addr - memory_inst->memory_data); + + return 0; +} + +bool +wasm_runtime_get_app_addr_range(WASMModuleInstanceCommon *module_inst_comm, + uint32 app_offset, uint32 *p_app_start_offset, + uint32 *p_app_end_offset) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + uint32 memory_data_size; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + return false; + } + + memory_data_size = memory_inst->memory_data_size; + + if (app_offset < memory_data_size) { + if (p_app_start_offset) + *p_app_start_offset = 0; + if (p_app_end_offset) + *p_app_end_offset = memory_data_size; + return true; + } + + return false; +} + +bool +wasm_runtime_get_native_addr_range(WASMModuleInstanceCommon *module_inst_comm, + uint8 *native_ptr, + uint8 **p_native_start_addr, + uint8 **p_native_end_addr) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + uint8 *addr = (uint8 *)native_ptr; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + return false; + } + + if (memory_inst->memory_data <= addr + && addr < memory_inst->memory_data_end) { + if (p_native_start_addr) + *p_native_start_addr = memory_inst->memory_data; + if (p_native_end_addr) + *p_native_end_addr = memory_inst->memory_data_end; + return true; + } + + return false; +} + +bool +wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, + uint32 app_buf_addr, uint32 app_buf_size, + void **p_native_addr) +{ + WASMMemoryInstance *memory_inst = wasm_get_default_memory(module_inst); + uint8 *native_addr; + + if (!memory_inst) { + goto fail; + } + + native_addr = memory_inst->memory_data + app_buf_addr; + + /* No need to check the app_offset and buf_size if memory access + boundary check with hardware trap is enabled */ +#ifndef OS_ENABLE_HW_BOUND_CHECK + if (app_buf_addr >= memory_inst->memory_data_size) { + goto fail; + } + + if (!is_str) { + if (app_buf_size > memory_inst->memory_data_size - app_buf_addr) { + goto fail; + } + } + else { + const char *str, *str_end; + + /* The whole string must be in the linear memory */ + str = (const char *)native_addr; + str_end = (const char *)memory_inst->memory_data_end; + while (str < str_end && *str != '\0') + str++; + if (str == str_end) + goto fail; + } +#endif + + *p_native_addr = (void *)native_addr; + return true; +fail: + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; +} + +WASMMemoryInstance * +wasm_get_default_memory(WASMModuleInstance *module_inst) +{ + if (module_inst->memories) + return module_inst->memories[0]; + else + return NULL; +} + +#ifndef OS_ENABLE_HW_BOUND_CHECK +bool +wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) +{ + WASMMemoryInstance *memory = wasm_get_default_memory(module); + uint8 *memory_data_old, *memory_data_new, *heap_data_old; + uint32 num_bytes_per_page, heap_size, total_size_old; + uint32 cur_page_count, max_page_count, total_page_count; + uint64 total_size_new; + bool ret = true; + + if (!memory) + return false; + + heap_data_old = memory->heap_data; + heap_size = (uint32)(memory->heap_data_end - memory->heap_data); + + memory_data_old = memory->memory_data; + total_size_old = memory->memory_data_size; + + num_bytes_per_page = memory->num_bytes_per_page; + cur_page_count = memory->cur_page_count; + max_page_count = memory->max_page_count; + total_page_count = inc_page_count + cur_page_count; + total_size_new = num_bytes_per_page * (uint64)total_page_count; + + if (inc_page_count <= 0) + /* No need to enlarge memory */ + return true; + + if (total_page_count < cur_page_count /* integer overflow */ + || total_page_count > max_page_count) { + return false; + } + + bh_assert(total_size_new <= 4 * (uint64)BH_GB); + if (total_size_new > UINT32_MAX) { + /* Resize to 1 page with size 4G-1 */ + num_bytes_per_page = UINT32_MAX; + total_page_count = max_page_count = 1; + total_size_new = UINT32_MAX; + } + +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (memory->is_shared) { + memory->num_bytes_per_page = num_bytes_per_page; + memory->cur_page_count = total_page_count; + memory->max_page_count = max_page_count; + /* No need to update memory->memory_data_size as it is + initialized with the maximum memory data size for + shared memory */ + return true; + } +#endif + + if (heap_size > 0) { + if (mem_allocator_is_heap_corrupted(memory->heap_handle)) { + wasm_runtime_show_app_heap_corrupted_prompt(); + return false; + } + } + + if (!(memory_data_new = + wasm_runtime_realloc(memory_data_old, (uint32)total_size_new))) { + if (!(memory_data_new = wasm_runtime_malloc((uint32)total_size_new))) { + return false; + } + if (memory_data_old) { + bh_memcpy_s(memory_data_new, (uint32)total_size_new, + memory_data_old, total_size_old); + wasm_runtime_free(memory_data_old); + } + } + + memset(memory_data_new + total_size_old, 0, + (uint32)total_size_new - total_size_old); + + if (heap_size > 0) { + if (mem_allocator_migrate(memory->heap_handle, + (char *)heap_data_old + + (memory_data_new - memory_data_old), + heap_size) + != 0) { + /* Don't return here as memory->memory_data is obsolete and + must be updated to be correctly used later. */ + ret = false; + } + } + + memory->heap_data = memory_data_new + (heap_data_old - memory_data_old); + memory->heap_data_end = memory->heap_data + heap_size; + + memory->num_bytes_per_page = num_bytes_per_page; + memory->cur_page_count = total_page_count; + memory->max_page_count = max_page_count; + memory->memory_data_size = (uint32)total_size_new; + + memory->memory_data = memory_data_new; + memory->memory_data_end = memory_data_new + (uint32)total_size_new; + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 || WASM_ENABLE_AOT != 0 +#if UINTPTR_MAX == UINT64_MAX + memory->mem_bound_check_1byte.u64 = total_size_new - 1; + memory->mem_bound_check_2bytes.u64 = total_size_new - 2; + memory->mem_bound_check_4bytes.u64 = total_size_new - 4; + memory->mem_bound_check_8bytes.u64 = total_size_new - 8; + memory->mem_bound_check_16bytes.u64 = total_size_new - 16; +#else + memory->mem_bound_check_1byte.u32[0] = (uint32)total_size_new - 1; + memory->mem_bound_check_2bytes.u32[0] = (uint32)total_size_new - 2; + memory->mem_bound_check_4bytes.u32[0] = (uint32)total_size_new - 4; + memory->mem_bound_check_8bytes.u32[0] = (uint32)total_size_new - 8; + memory->mem_bound_check_16bytes.u32[0] = (uint32)total_size_new - 16; +#endif +#endif + + return ret; +} +#else +bool +wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) +{ + WASMMemoryInstance *memory = wasm_get_default_memory(module); + uint32 num_bytes_per_page, total_size_old; + uint32 cur_page_count, max_page_count, total_page_count; + uint64 total_size_new; + + if (!memory) + return false; + + num_bytes_per_page = memory->num_bytes_per_page; + cur_page_count = memory->cur_page_count; + max_page_count = memory->max_page_count; + total_size_old = num_bytes_per_page * cur_page_count; + total_page_count = inc_page_count + cur_page_count; + total_size_new = num_bytes_per_page * (uint64)total_page_count; + + if (inc_page_count <= 0) + /* No need to enlarge memory */ + return true; + + if (total_page_count < cur_page_count /* integer overflow */ + || total_page_count > max_page_count) { + return false; + } + + bh_assert(total_size_new <= 4 * (uint64)BH_GB); + if (total_size_new > UINT32_MAX) { + /* Resize to 1 page with size 4G-1 */ + num_bytes_per_page = UINT32_MAX; + total_page_count = max_page_count = 1; + total_size_new = UINT32_MAX; + } + +#ifdef BH_PLATFORM_WINDOWS + if (!os_mem_commit(memory->memory_data_end, + (uint32)total_size_new - total_size_old, + MMAP_PROT_READ | MMAP_PROT_WRITE)) { + return false; + } +#endif + + if (os_mprotect(memory->memory_data_end, + (uint32)total_size_new - total_size_old, + MMAP_PROT_READ | MMAP_PROT_WRITE) + != 0) { +#ifdef BH_PLATFORM_WINDOWS + os_mem_decommit(memory->memory_data_end, + (uint32)total_size_new - total_size_old); +#endif + return false; + } + + /* The increased pages are filled with zero by the OS when os_mmap, + no need to memset it again here */ + + memory->num_bytes_per_page = num_bytes_per_page; + memory->cur_page_count = total_page_count; + memory->max_page_count = max_page_count; + memory->memory_data_size = (uint32)total_size_new; + memory->memory_data_end = memory->memory_data + (uint32)total_size_new; + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 || WASM_ENABLE_AOT != 0 + memory->mem_bound_check_1byte.u64 = total_size_new - 1; + memory->mem_bound_check_2bytes.u64 = total_size_new - 2; + memory->mem_bound_check_4bytes.u64 = total_size_new - 4; + memory->mem_bound_check_8bytes.u64 = total_size_new - 8; + memory->mem_bound_check_16bytes.u64 = total_size_new - 16; +#endif + + return true; +} +#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index 409ea0ff..cd4ecb70 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -135,43 +135,117 @@ static os_thread_local_attribute WASMExecEnv *exec_env_tls = NULL; static void runtime_signal_handler(void *sig_addr) { - WASMModuleInstanceCommon *module_inst; - WASMSignalInfo sig_info; + WASMModuleInstance *module_inst; + WASMMemoryInstance *memory_inst; + WASMJmpBuf *jmpbuf_node; + uint8 *mapped_mem_start_addr = NULL; + uint8 *mapped_mem_end_addr = NULL; + uint8 *stack_min_addr; + uint32 page_size; + uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; - sig_info.exec_env_tls = exec_env_tls; - sig_info.sig_addr = sig_addr; - if (exec_env_tls) { - module_inst = exec_env_tls->module_inst; -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - wasm_signal_handler(&sig_info); -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - aot_signal_handler(&sig_info); -#endif + /* Check whether current thread is running wasm function */ + if (exec_env_tls && exec_env_tls->handle == os_self_thread() + && (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) { + /* Get mapped mem info of current instance */ + module_inst = (WASMModuleInstance *)exec_env_tls->module_inst; + /* Get the default memory instance */ + memory_inst = wasm_get_default_memory(module_inst); + if (memory_inst) { + mapped_mem_start_addr = memory_inst->memory_data; + mapped_mem_end_addr = memory_inst->memory_data + 8 * (uint64)BH_GB; + } + + /* Get stack info of current thread */ + page_size = os_getpagesize(); + stack_min_addr = os_thread_get_stack_boundary(); + + if (memory_inst + && (mapped_mem_start_addr <= (uint8 *)sig_addr + && (uint8 *)sig_addr < mapped_mem_end_addr)) { + /* The address which causes segmentation fault is inside + the memory instance's guard regions */ + wasm_set_exception(module_inst, "out of bounds memory access"); + os_longjmp(jmpbuf_node->jmpbuf, 1); + } + else if (stack_min_addr - page_size <= (uint8 *)sig_addr + && (uint8 *)sig_addr + < stack_min_addr + page_size * guard_page_count) { + /* The address which causes segmentation fault is inside + native thread's guard page */ + wasm_set_exception(module_inst, "native stack overflow"); + os_longjmp(jmpbuf_node->jmpbuf, 1); + } } } #else static LONG runtime_exception_handler(EXCEPTION_POINTERS *exce_info) { - WASMModuleInstanceCommon *module_inst; - WASMSignalInfo sig_info; + PEXCEPTION_RECORD ExceptionRecord = exce_info->ExceptionRecord; + uint8 *sig_addr = (uint8 *)ExceptionRecord->ExceptionInformation[1]; + WASMModuleInstance *module_inst; + WASMMemoryInstance *memory_inst; + WASMJmpBuf *jmpbuf_node; + uint8 *mapped_mem_start_addr = NULL; + uint8 *mapped_mem_end_addr = NULL; + uint32 page_size = os_getpagesize(); - sig_info.exec_env_tls = exec_env_tls; - sig_info.exce_info = exce_info; - if (exec_env_tls) { - module_inst = exec_env_tls->module_inst; -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - return wasm_exception_handler(&sig_info); -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - return aot_exception_handler(&sig_info); -#endif + if (exec_env_tls && exec_env_tls->handle == os_self_thread() + && (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) { + module_inst = (WASMModuleInstance *)exec_env_tls->module_inst; + if (ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { + /* Get the default memory instance */ + memory_inst = wasm_get_default_memory(module_inst); + if (memory_inst) { + mapped_mem_start_addr = memory_inst->memory_data; + mapped_mem_end_addr = + memory_inst->memory_data + 8 * (uint64)BH_GB; + if (mapped_mem_start_addr <= (uint8 *)sig_addr + && (uint8 *)sig_addr < mapped_mem_end_addr) { + /* The address which causes segmentation fault is inside + the memory instance's guard regions. + Set exception and let the wasm func continue to run, when + the wasm func returns, the caller will check whether the + exception is thrown and return to runtime. */ + wasm_set_exception(module_inst, + "out of bounds memory access"); + if (module_inst->module_type == Wasm_Module_Bytecode) { + /* Continue to search next exception handler for + interpreter mode as it can be caught by + `__try { .. } __except { .. }` sentences in + wasm_runtime.c */ + return EXCEPTION_CONTINUE_SEARCH; + } + else { + /* Skip current instruction and continue to run for + AOT mode. TODO: implement unwind support for AOT + code in Windows platform */ + exce_info->ContextRecord->Rip++; + return EXCEPTION_CONTINUE_EXECUTION; + } + } + } + } + else if (ExceptionRecord->ExceptionCode == EXCEPTION_STACK_OVERFLOW) { + /* Set stack overflow exception and let the wasm func continue + to run, when the wasm func returns, the caller will check + whether the exception is thrown and return to runtime, and + the damaged stack will be recovered by _resetstkoflw(). */ + wasm_set_exception(module_inst, "native stack overflow"); + if (module_inst->module_type == Wasm_Module_Bytecode) { + return EXCEPTION_CONTINUE_SEARCH; + } + else { + return EXCEPTION_CONTINUE_EXECUTION; + } + } } + + os_printf("Unhandled exception thrown: exception code: 0x%lx, " + "exception address: %p, exception information: %p\n", + ExceptionRecord->ExceptionCode, ExceptionRecord->ExceptionAddress, + sig_addr); return EXCEPTION_CONTINUE_SEARCH; } #endif /* end of BH_PLATFORM_WINDOWS */ @@ -912,22 +986,7 @@ wasm_runtime_load(uint8 *buf, uint32 size, char *error_buf, WASMModuleCommon *module_common = NULL; if (get_package_type(buf, size) == Wasm_Module_Bytecode) { -#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_JIT != 0 - AOTModule *aot_module; - WASMModule *module = wasm_load(buf, size, error_buf, error_buf_size); - if (!module) - return NULL; - - if (!(aot_module = - aot_convert_wasm_module(module, error_buf, error_buf_size))) { - wasm_unload(module); - return NULL; - } - - module_common = (WASMModuleCommon *)aot_module; - return register_module_with_null_name(module_common, error_buf, - error_buf_size); -#elif WASM_ENABLE_INTERP != 0 +#if WASM_ENABLE_INTERP != 0 module_common = (WASMModuleCommon *)wasm_load(buf, size, error_buf, error_buf_size); return register_module_with_null_name(module_common, error_buf, @@ -1235,19 +1294,18 @@ wasm_runtime_dump_mem_consumption(WASMExecEnv *exec_env) &module_inst_mem_consps); wasm_get_module_mem_consumption(wasm_module, &module_mem_consps); if (wasm_module_inst->module->aux_stack_top_global_index != (uint32)-1) - max_aux_stack_used = wasm_module_inst->max_aux_stack_used; + max_aux_stack_used = wasm_module_inst->e->max_aux_stack_used; } #endif #if WASM_ENABLE_AOT != 0 if (module_inst_common->module_type == Wasm_Module_AoT) { AOTModuleInstance *aot_module_inst = (AOTModuleInstance *)module_inst_common; - AOTModule *aot_module = (AOTModule *)aot_module_inst->aot_module.ptr; + AOTModule *aot_module = (AOTModule *)aot_module_inst->module; module_common = (WASMModuleCommon *)aot_module; - if (aot_module_inst->memories.ptr) { - AOTMemoryInstance **memories = - (AOTMemoryInstance **)aot_module_inst->memories.ptr; - heap_handle = memories[0]->heap_handle.ptr; + if (aot_module_inst->memories) { + AOTMemoryInstance **memories = aot_module_inst->memories; + heap_handle = memories[0]->heap_handle; } aot_get_module_inst_mem_consumption(aot_module_inst, &module_inst_mem_consps); @@ -1982,99 +2040,130 @@ fail1: } bool -wasm_runtime_create_exec_env_singleton(WASMModuleInstanceCommon *module_inst) +wasm_runtime_create_exec_env_singleton( + WASMModuleInstanceCommon *module_inst_comm) { -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - return wasm_create_exec_env_singleton( - (WASMModuleInstance *)module_inst); -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - return aot_create_exec_env_singleton((AOTModuleInstance *)module_inst); -#endif - return false; + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMExecEnv *exec_env = NULL; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + if (module_inst->exec_env_singleton) { + return true; + } + + exec_env = wasm_exec_env_create(module_inst_comm, + module_inst->default_wasm_stack_size); + if (exec_env) + module_inst->exec_env_singleton = exec_env; + + return exec_env ? true : false; } WASMExecEnv * -wasm_runtime_get_exec_env_singleton(WASMModuleInstanceCommon *module_inst) +wasm_runtime_get_exec_env_singleton(WASMModuleInstanceCommon *module_inst_comm) { -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) { - if (!((WASMModuleInstance *)module_inst)->exec_env_singleton) { - wasm_create_exec_env_singleton((WASMModuleInstance *)module_inst); - } - return ((WASMModuleInstance *)module_inst)->exec_env_singleton; + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + if (!module_inst->exec_env_singleton) { + wasm_runtime_create_exec_env_singleton(module_inst_comm); } -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) { - if (!((AOTModuleInstance *)module_inst)->exec_env_singleton.ptr) { - aot_create_exec_env_singleton((AOTModuleInstance *)module_inst); - } - return (WASMExecEnv *)((AOTModuleInstance *)module_inst) - ->exec_env_singleton.ptr; - } -#endif - return NULL; + return module_inst->exec_env_singleton; } void -wasm_runtime_set_exception(WASMModuleInstanceCommon *module_inst, - const char *exception) +wasm_set_exception(WASMModuleInstance *module_inst, const char *exception) { -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) { - wasm_set_exception((WASMModuleInstance *)module_inst, exception); - return; - } -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) { - aot_set_exception((AOTModuleInstance *)module_inst, exception); - return; - } -#endif + if (exception) + snprintf(module_inst->cur_exception, sizeof(module_inst->cur_exception), + "Exception: %s", exception); + else + module_inst->cur_exception[0] = '\0'; +} + +/* clang-format off */ +static const char *exception_msgs[] = { + "unreachable", /* EXCE_UNREACHABLE */ + "allocate memory failed", /* EXCE_OUT_OF_MEMORY */ + "out of bounds memory access", /* EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS */ + "integer overflow", /* EXCE_INTEGER_OVERFLOW */ + "integer divide by zero", /* EXCE_INTEGER_DIVIDE_BY_ZERO */ + "invalid conversion to integer", /* EXCE_INVALID_CONVERSION_TO_INTEGER */ + "indirect call type mismatch", /* EXCE_INVALID_FUNCTION_TYPE_INDEX */ + "invalid function index", /* EXCE_INVALID_FUNCTION_INDEX */ + "undefined element", /* EXCE_UNDEFINED_ELEMENT */ + "uninitialized element", /* EXCE_UNINITIALIZED_ELEMENT */ + "failed to call unlinked import function", /* EXCE_CALL_UNLINKED_IMPORT_FUNC */ + "native stack overflow", /* EXCE_NATIVE_STACK_OVERFLOW */ + "unaligned atomic", /* EXCE_UNALIGNED_ATOMIC */ + "wasm auxiliary stack overflow", /* EXCE_AUX_STACK_OVERFLOW */ + "wasm auxiliary stack underflow", /* EXCE_AUX_STACK_UNDERFLOW */ + "out of bounds table access", /* EXCE_OUT_OF_BOUNDS_TABLE_ACCESS */ + "wasm operand stack overflow", /* EXCE_OPERAND_STACK_OVERFLOW */ + "", /* EXCE_ALREADY_THROWN */ +}; +/* clang-format on */ + +void +wasm_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id) +{ + if (id < EXCE_NUM) + wasm_set_exception(module_inst, exception_msgs[id]); + else + wasm_set_exception(module_inst, "unknown exception"); } const char * -wasm_runtime_get_exception(WASMModuleInstanceCommon *module_inst) +wasm_get_exception(WASMModuleInstance *module_inst) { -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) { - return wasm_get_exception((WASMModuleInstance *)module_inst); - } -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) { - return aot_get_exception((AOTModuleInstance *)module_inst); - } -#endif - return NULL; + if (module_inst->cur_exception[0] == '\0') + return NULL; + else + return module_inst->cur_exception; } void -wasm_runtime_clear_exception(WASMModuleInstanceCommon *module_inst) +wasm_runtime_set_exception(WASMModuleInstanceCommon *module_inst_comm, + const char *exception) { - wasm_runtime_set_exception(module_inst, NULL); + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + wasm_set_exception(module_inst, exception); +} + +const char * +wasm_runtime_get_exception(WASMModuleInstanceCommon *module_inst_comm) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + return wasm_get_exception(module_inst); } void -wasm_runtime_set_custom_data_internal(WASMModuleInstanceCommon *module_inst, - void *custom_data) +wasm_runtime_clear_exception(WASMModuleInstanceCommon *module_inst_comm) { -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) { - ((WASMModuleInstance *)module_inst)->custom_data = custom_data; - return; - } -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) { - ((AOTModuleInstance *)module_inst)->custom_data.ptr = custom_data; - return; - } -#endif + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + wasm_runtime_set_exception(module_inst_comm, NULL); +} + +void +wasm_runtime_set_custom_data_internal( + WASMModuleInstanceCommon *module_inst_comm, void *custom_data) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + module_inst->custom_data = custom_data; } void @@ -2089,17 +2178,13 @@ wasm_runtime_set_custom_data(WASMModuleInstanceCommon *module_inst, } void * -wasm_runtime_get_custom_data(WASMModuleInstanceCommon *module_inst) +wasm_runtime_get_custom_data(WASMModuleInstanceCommon *module_inst_comm) { -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - return ((WASMModuleInstance *)module_inst)->custom_data; -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - return ((AOTModuleInstance *)module_inst)->custom_data.ptr; -#endif - return NULL; + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + return module_inst->custom_data; } uint32 @@ -2171,155 +2256,6 @@ wasm_runtime_module_dup_data(WASMModuleInstanceCommon *module_inst, return 0; } -bool -wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst, - uint32 app_offset, uint32 size) -{ -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - return wasm_validate_app_addr((WASMModuleInstance *)module_inst, - app_offset, size); -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - return aot_validate_app_addr((AOTModuleInstance *)module_inst, - app_offset, size); -#endif - return false; -} - -bool -wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst, - uint32 app_str_offset) -{ - uint32 app_end_offset; - char *str, *str_end; - - if (!wasm_runtime_get_app_addr_range(module_inst, app_str_offset, NULL, - &app_end_offset)) - goto fail; - - str = wasm_runtime_addr_app_to_native(module_inst, app_str_offset); - str_end = str + (app_end_offset - app_str_offset); - while (str < str_end && *str != '\0') - str++; - if (str == str_end) - goto fail; - return true; - -fail: - wasm_runtime_set_exception(module_inst, "out of bounds memory access"); - return false; -} - -bool -wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst, - void *native_ptr, uint32 size) -{ -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - return wasm_validate_native_addr((WASMModuleInstance *)module_inst, - native_ptr, size); -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - return aot_validate_native_addr((AOTModuleInstance *)module_inst, - native_ptr, size); -#endif - return false; -} - -void * -wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst, - uint32 app_offset) -{ -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - return wasm_addr_app_to_native((WASMModuleInstance *)module_inst, - app_offset); -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - return aot_addr_app_to_native((AOTModuleInstance *)module_inst, - app_offset); -#endif - return NULL; -} - -uint32 -wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst, - void *native_ptr) -{ -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - return wasm_addr_native_to_app((WASMModuleInstance *)module_inst, - native_ptr); -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - return aot_addr_native_to_app((AOTModuleInstance *)module_inst, - native_ptr); -#endif - return 0; -} - -bool -wasm_runtime_get_app_addr_range(WASMModuleInstanceCommon *module_inst, - uint32 app_offset, uint32 *p_app_start_offset, - uint32 *p_app_end_offset) -{ -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - return wasm_get_app_addr_range((WASMModuleInstance *)module_inst, - app_offset, p_app_start_offset, - p_app_end_offset); -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - return aot_get_app_addr_range((AOTModuleInstance *)module_inst, - app_offset, p_app_start_offset, - p_app_end_offset); -#endif - return false; -} - -bool -wasm_runtime_get_native_addr_range(WASMModuleInstanceCommon *module_inst, - uint8 *native_ptr, - uint8 **p_native_start_addr, - uint8 **p_native_end_addr) -{ -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - return wasm_get_native_addr_range((WASMModuleInstance *)module_inst, - native_ptr, p_native_start_addr, - p_native_end_addr); -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - return aot_get_native_addr_range((AOTModuleInstance *)module_inst, - native_ptr, p_native_start_addr, - p_native_end_addr); -#endif - return false; -} - -bool -wasm_runtime_enlarge_memory(WASMModuleInstanceCommon *module, - uint32 inc_page_count) -{ -#if WASM_ENABLE_INTERP != 0 - if (module->module_type == Wasm_Module_Bytecode) - return wasm_enlarge_memory((WASMModuleInstance *)module, - inc_page_count); -#endif -#if WASM_ENABLE_AOT != 0 - if (module->module_type == Wasm_Module_AoT) - return aot_enlarge_memory((AOTModuleInstance *)module, inc_page_count); -#endif - return false; -} - #if WASM_ENABLE_LIBC_WASI != 0 static WASIArguments * @@ -2477,19 +2413,6 @@ wasm_runtime_init_wasi(WASMModuleInstanceCommon *module_inst, wasm_runtime_set_wasi_ctx(module_inst, wasi_ctx); -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode - && !((WASMModuleInstance *)module_inst)->default_memory) - return true; -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT - && !((AOTModuleInstance *)module_inst) - ->global_table_data.memory_instances[0] - .memory_data.ptr) - return true; -#endif - /* process argv[0], trip the path and suffix, only keep the program name */ if (!copy_string_array((const char **)argv, argc, &argv_buf, &argv_list, &argv_buf_size)) { @@ -2796,7 +2719,7 @@ wasm_runtime_is_wasi_mode(WASMModuleInstanceCommon *module_inst) #endif #if WASM_ENABLE_AOT != 0 if (module_inst->module_type == Wasm_Module_AoT - && ((AOTModule *)((AOTModuleInstance *)module_inst)->aot_module.ptr) + && ((AOTModule *)((AOTModuleInstance *)module_inst)->module) ->import_wasi_api) return true; #endif @@ -2832,7 +2755,7 @@ wasm_runtime_lookup_wasi_start_function(WASMModuleInstanceCommon *module_inst) if (module_inst->module_type == Wasm_Module_AoT) { AOTModuleInstance *aot_inst = (AOTModuleInstance *)module_inst; AOTFunctionInstance *export_funcs = - (AOTFunctionInstance *)aot_inst->export_funcs.ptr; + (AOTFunctionInstance *)aot_inst->export_functions; for (i = 0; i < aot_inst->export_func_count; i++) { if (!strcmp(export_funcs[i].func_name, "_start")) { AOTFuncType *func_type = export_funcs[i].u.func.func_type; @@ -2905,49 +2828,37 @@ wasm_runtime_destroy_wasi(WASMModuleInstanceCommon *module_inst) #endif WASIContext * -wasm_runtime_get_wasi_ctx(WASMModuleInstanceCommon *module_inst) +wasm_runtime_get_wasi_ctx(WASMModuleInstanceCommon *module_inst_comm) { -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - return ((WASMModuleInstance *)module_inst)->wasi_ctx; -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - return ((AOTModuleInstance *)module_inst)->wasi_ctx.ptr; -#endif - return NULL; + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + return module_inst->wasi_ctx; } void -wasm_runtime_set_wasi_ctx(WASMModuleInstanceCommon *module_inst, +wasm_runtime_set_wasi_ctx(WASMModuleInstanceCommon *module_inst_comm, WASIContext *wasi_ctx) { -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - ((WASMModuleInstance *)module_inst)->wasi_ctx = wasi_ctx; -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - ((AOTModuleInstance *)module_inst)->wasi_ctx.ptr = wasi_ctx; -#endif + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + module_inst->wasi_ctx = wasi_ctx; } #endif /* end of WASM_ENABLE_LIBC_WASI */ WASMModuleCommon * wasm_exec_env_get_module(WASMExecEnv *exec_env) { - WASMModuleInstanceCommon *module_inst = + WASMModuleInstanceCommon *module_inst_comm = wasm_runtime_get_module_inst(exec_env); -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - return (WASMModuleCommon *)((WASMModuleInstance *)module_inst)->module; -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - return (WASMModuleCommon *)((AOTModuleInstance *)module_inst) - ->aot_module.ptr; -#endif - return NULL; + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + return (WASMModuleCommon *)module_inst->module; } #if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 @@ -4510,8 +4421,8 @@ interp_mark_all_externrefs(WASMModuleInstance *module_inst) WASMGlobalInstance *global; WASMTableInstance *table; - global = module_inst->globals; - for (i = 0; i < module_inst->global_count; i++, global++) { + global = module_inst->e->globals; + for (i = 0; i < module_inst->e->global_count; i++, global++) { if (global->type == VALUE_TYPE_EXTERNREF) { externref_idx = *(uint32 *)(global_data + global->data_offset); mark_externref(externref_idx); @@ -4519,14 +4430,23 @@ interp_mark_all_externrefs(WASMModuleInstance *module_inst) } for (i = 0; i < module_inst->table_count; i++) { + uint8 elem_type = 0; + uint32 init_size, max_size; + table = wasm_get_table_inst(module_inst, i); - if (table->elem_type == VALUE_TYPE_EXTERNREF) { - table_data = (uint32 *)table->base_addr; + (void)wasm_runtime_get_table_inst_elem_type( + (WASMModuleInstanceCommon *)module_inst, i, &elem_type, &init_size, + &max_size); + + if (elem_type == VALUE_TYPE_EXTERNREF) { + table_data = table->elems; for (j = 0; j < table->cur_size; j++) { externref_idx = table_data[j]; mark_externref(externref_idx); } } + (void)init_size; + (void)max_size; } } #endif @@ -4536,25 +4456,23 @@ static void aot_mark_all_externrefs(AOTModuleInstance *module_inst) { uint32 i = 0, j = 0; - const AOTModule *module = (AOTModule *)(module_inst->aot_module.ptr); + const AOTModule *module = (AOTModule *)module_inst->module; const AOTTable *table = module->tables; const AOTGlobal *global = module->globals; - const AOTTableInstance *table_inst = - (AOTTableInstance *)module_inst->tables.ptr; + const AOTTableInstance *table_inst; for (i = 0; i < module->global_count; i++, global++) { if (global->type == VALUE_TYPE_EXTERNREF) { - mark_externref(*(uint32 *)((uint8 *)module_inst->global_data.ptr - + global->data_offset)); + mark_externref( + *(uint32 *)(module_inst->global_data + global->data_offset)); } } - for (i = 0; i < module->table_count; - i++, table_inst = aot_next_tbl_inst(table_inst)) { - + for (i = 0; i < module->table_count; i++) { + table_inst = module_inst->tables[i]; if ((table + i)->elem_type == VALUE_TYPE_EXTERNREF) { while (j < table_inst->cur_size) { - mark_externref(table_inst->data[j++]); + mark_externref(table_inst->elems[j++]); } } } @@ -4707,6 +4625,82 @@ wasm_runtime_dump_call_stack_to_buf(wasm_exec_env_t exec_env, char *buf, } #endif /* end of WASM_ENABLE_DUMP_CALL_STACK */ +bool +wasm_runtime_get_table_elem_type(const WASMModuleCommon *module_comm, + uint32 table_idx, uint8 *out_elem_type, + uint32 *out_min_size, uint32 *out_max_size) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_comm->module_type == Wasm_Module_Bytecode) { + WASMModule *module = (WASMModule *)module_comm; + + if (table_idx < module->import_table_count) { + WASMTableImport *import_table = + &((module->import_tables + table_idx)->u.table); + *out_elem_type = import_table->elem_type; + *out_min_size = import_table->init_size; + *out_max_size = import_table->max_size; + } + else { + WASMTable *table = + module->tables + (table_idx - module->import_table_count); + *out_elem_type = table->elem_type; + *out_min_size = table->init_size; + *out_max_size = table->max_size; + } + return true; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_comm->module_type == Wasm_Module_AoT) { + AOTModule *module = (AOTModule *)module_comm; + + if (table_idx < module->import_table_count) { + AOTImportTable *import_table = module->import_tables + table_idx; + *out_elem_type = VALUE_TYPE_FUNCREF; + *out_min_size = import_table->table_init_size; + *out_max_size = import_table->table_max_size; + } + else { + AOTTable *table = + module->tables + (table_idx - module->import_table_count); + *out_elem_type = table->elem_type; + *out_min_size = table->table_init_size; + *out_max_size = table->table_max_size; + } + return true; + } +#endif + + return false; +} + +bool +wasm_runtime_get_table_inst_elem_type( + const WASMModuleInstanceCommon *module_inst_comm, uint32 table_idx, + uint8 *out_elem_type, uint32 *out_min_size, uint32 *out_max_size) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst_comm->module_type == Wasm_Module_Bytecode) { + WASMModuleInstance *module_inst = + (WASMModuleInstance *)module_inst_comm; + return wasm_runtime_get_table_elem_type( + (WASMModuleCommon *)module_inst->module, table_idx, out_elem_type, + out_min_size, out_max_size); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst_comm->module_type == Wasm_Module_AoT) { + AOTModuleInstance *module_inst = (AOTModuleInstance *)module_inst_comm; + return wasm_runtime_get_table_elem_type( + (WASMModuleCommon *)module_inst->module, table_idx, out_elem_type, + out_min_size, out_max_size); + } +#endif + return false; +} + bool wasm_runtime_get_export_func_type(const WASMModuleCommon *module_comm, const WASMExport *export, WASMType **out) @@ -4847,50 +4841,8 @@ wasm_runtime_get_export_table_type(const WASMModuleCommon *module_comm, uint8 *out_elem_type, uint32 *out_min_size, uint32 *out_max_size) { -#if WASM_ENABLE_INTERP != 0 - if (module_comm->module_type == Wasm_Module_Bytecode) { - WASMModule *module = (WASMModule *)module_comm; - - if (export->index < module->import_table_count) { - WASMTableImport *import_table = - &((module->import_tables + export->index)->u.table); - *out_elem_type = import_table->elem_type; - *out_min_size = import_table->init_size; - *out_max_size = import_table->max_size; - } - else { - WASMTable *table = - module->tables + (export->index - module->import_table_count); - *out_elem_type = table->elem_type; - *out_min_size = table->init_size; - *out_max_size = table->max_size; - } - return true; - } -#endif - -#if WASM_ENABLE_AOT != 0 - if (module_comm->module_type == Wasm_Module_AoT) { - AOTModule *module = (AOTModule *)module_comm; - - if (export->index < module->import_table_count) { - AOTImportTable *import_table = - module->import_tables + export->index; - *out_elem_type = VALUE_TYPE_FUNCREF; - *out_min_size = import_table->table_init_size; - *out_max_size = import_table->table_max_size; - } - else { - AOTTable *table = - module->tables + (export->index - module->import_table_count); - *out_elem_type = table->elem_type; - *out_min_size = table->table_init_size; - *out_max_size = table->table_max_size; - } - return true; - } -#endif - return false; + return wasm_runtime_get_table_elem_type( + module_comm, export->index, out_elem_type, out_min_size, out_max_size); } static inline bool @@ -5036,10 +4988,13 @@ wasm_runtime_invoke_c_api_native(WASMModuleInstanceCommon *module_inst, if (trap) { if (trap->message->data) { /* since trap->message->data does not end with '\0' */ - char trap_message[128] = { 0 }; - bh_memcpy_s(trap_message, 127, trap->message->data, - (trap->message->size < 127 ? (uint32)trap->message->size - : 127)); + char trap_message[108] = { 0 }; + uint32 max_size_to_copy = (uint32)sizeof(trap_message) - 1; + uint32 size_to_copy = (trap->message->size < max_size_to_copy) + ? (uint32)trap->message->size + : max_size_to_copy; + bh_memcpy_s(trap_message, (uint32)sizeof(trap_message), + trap->message->data, size_to_copy); wasm_runtime_set_exception(module_inst, trap_message); } else { diff --git a/core/iwasm/common/wasm_runtime_common.h b/core/iwasm/common/wasm_runtime_common.h index 87473397..e05ada14 100644 --- a/core/iwasm/common/wasm_runtime_common.h +++ b/core/iwasm/common/wasm_runtime_common.h @@ -852,17 +852,6 @@ wasm_runtime_dump_line_buf_impl(const char *line_buf, bool dump_or_print, WASMModuleCommon * wasm_exec_env_get_module(WASMExecEnv *exec_env); -/** - * Enlarge wasm memory data space. - * - * @param module the wasm module instance - * @param inc_page_count denote the page number to increase - * @return return true if enlarge successfully, false otherwise - */ -bool -wasm_runtime_enlarge_memory(WASMModuleInstanceCommon *module, - uint32 inc_page_count); - /* See wasm_export.h for description */ WASM_RUNTIME_API_EXTERN bool wasm_runtime_register_natives(const char *module_name, @@ -900,6 +889,16 @@ wasm_runtime_dump_module_inst_mem_consumption( void wasm_runtime_dump_exec_env_mem_consumption(const WASMExecEnv *exec_env); +bool +wasm_runtime_get_table_elem_type(const WASMModuleCommon *module_comm, + uint32 table_idx, uint8 *out_elem_type, + uint32 *out_min_size, uint32 *out_max_size); + +bool +wasm_runtime_get_table_inst_elem_type( + const WASMModuleInstanceCommon *module_inst_comm, uint32 table_idx, + uint8 *out_elem_type, uint32 *out_min_size, uint32 *out_max_size); + bool wasm_runtime_get_export_func_type(const WASMModuleCommon *module_comm, const WASMExport *export_, WASMType **out); diff --git a/core/iwasm/common/wasm_shared_memory.c b/core/iwasm/common/wasm_shared_memory.c index 395c677f..4ae88e89 100644 --- a/core/iwasm/common/wasm_shared_memory.c +++ b/core/iwasm/common/wasm_shared_memory.c @@ -314,44 +314,26 @@ uint32 wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address, uint64 expect, int64 timeout, bool wait64) { + WASMModuleInstance *module_inst = (WASMModuleInstance *)module; AtomicWaitInfo *wait_info; AtomicWaitNode *wait_node; bool check_ret, is_timeout; -#if WASM_ENABLE_INTERP != 0 - if (module->module_type == Wasm_Module_Bytecode) { - WASMModuleInstance *module_inst = (WASMModuleInstance *)module; - /* Currently we have only one memory instance */ - if (!module_inst->memories[0]->is_shared) { - wasm_runtime_set_exception(module, "expected shared memory"); - return -1; - } - if ((uint8 *)address < module_inst->memories[0]->memory_data - || (uint8 *)address + (wait64 ? 8 : 4) - > module_inst->memories[0]->memory_data_end) { - wasm_runtime_set_exception(module, "out of bounds memory access"); - return -1; - } + bh_assert(module->module_type == Wasm_Module_Bytecode + || module->module_type == Wasm_Module_AoT); + + /* Currently we have only one memory instance */ + if (!module_inst->memories[0]->is_shared) { + wasm_runtime_set_exception(module, "expected shared memory"); + return -1; } -#endif -#if WASM_ENABLE_AOT != 0 - if (module->module_type == Wasm_Module_AoT) { - AOTModuleInstance *aot_inst = (AOTModuleInstance *)module; - AOTMemoryInstance *aot_memory = - ((AOTMemoryInstance **)aot_inst->memories.ptr)[0]; - /* Currently we have only one memory instance */ - if (!aot_memory->is_shared) { - wasm_runtime_set_exception(module, "expected shared memory"); - return -1; - } - if ((uint8 *)address < (uint8 *)aot_memory->memory_data.ptr - || (uint8 *)address + (wait64 ? 8 : 4) - > (uint8 *)aot_memory->memory_data_end.ptr) { - wasm_runtime_set_exception(module, "out of bounds memory access"); - return -1; - } + + if ((uint8 *)address < module_inst->memories[0]->memory_data + || (uint8 *)address + (wait64 ? 8 : 4) + > module_inst->memories[0]->memory_data_end) { + wasm_runtime_set_exception(module, "out of bounds memory access"); + return -1; } -#endif /* acquire the wait info, create new one if not exists */ wait_info = acquire_wait_info(address, true); @@ -433,33 +415,18 @@ uint32 wasm_runtime_atomic_notify(WASMModuleInstanceCommon *module, void *address, uint32 count) { + WASMModuleInstance *module_inst = (WASMModuleInstance *)module; uint32 notify_result; AtomicWaitInfo *wait_info; -#if WASM_ENABLE_INTERP != 0 - if (module->module_type == Wasm_Module_Bytecode) { - WASMModuleInstance *module_inst = (WASMModuleInstance *)module; - if ((uint8 *)address < module_inst->memories[0]->memory_data - || (uint8 *)address + 4 - > module_inst->memories[0]->memory_data_end) { - wasm_runtime_set_exception(module, "out of bounds memory access"); - return -1; - } + bh_assert(module->module_type == Wasm_Module_Bytecode + || module->module_type == Wasm_Module_AoT); + + if ((uint8 *)address < module_inst->memories[0]->memory_data + || (uint8 *)address + 4 > module_inst->memories[0]->memory_data_end) { + wasm_runtime_set_exception(module, "out of bounds memory access"); + return -1; } -#endif -#if WASM_ENABLE_AOT != 0 - if (module->module_type == Wasm_Module_AoT) { - AOTModuleInstance *aot_inst = (AOTModuleInstance *)module; - AOTMemoryInstance *aot_memory = - ((AOTMemoryInstance **)aot_inst->memories.ptr)[0]; - if ((uint8 *)address < (uint8 *)aot_memory->memory_data.ptr - || (uint8 *)address + 4 - > (uint8 *)aot_memory->memory_data_end.ptr) { - wasm_runtime_set_exception(module, "out of bounds memory access"); - return -1; - } - } -#endif wait_info = acquire_wait_info(address, false); diff --git a/core/iwasm/compilation/aot.h b/core/iwasm/compilation/aot.h index a94bf646..8e8598c7 100644 --- a/core/iwasm/compilation/aot.h +++ b/core/iwasm/compilation/aot.h @@ -304,14 +304,22 @@ aot_set_last_error_v(const char *format, ...); #endif static inline uint32 -aot_get_tbl_data_slots(const AOTTable *tbl) +aot_get_imp_tbl_data_slots(const AOTImportTable *tbl, bool is_jit_mode) { +#if WASM_ENABLE_MULTI_MODULE != 0 + if (is_jit_mode) + return tbl->table_max_size; +#endif return tbl->possible_grow ? tbl->table_max_size : tbl->table_init_size; } static inline uint32 -aot_get_imp_tbl_data_slots(const AOTImportTable *tbl) +aot_get_tbl_data_slots(const AOTTable *tbl, bool is_jit_mode) { +#if WASM_ENABLE_MULTI_MODULE != 0 + if (is_jit_mode) + return tbl->table_max_size; +#endif return tbl->possible_grow ? tbl->table_max_size : tbl->table_init_size; } diff --git a/core/iwasm/compilation/aot_compiler.c b/core/iwasm/compilation/aot_compiler.c index 04425154..527c1a50 100644 --- a/core/iwasm/compilation/aot_compiler.c +++ b/core/iwasm/compilation/aot_compiler.c @@ -211,6 +211,7 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) case WASM_OP_BLOCK: case WASM_OP_LOOP: case WASM_OP_IF: + { value_type = *frame_ip++; if (value_type == VALUE_TYPE_I32 || value_type == VALUE_TYPE_I64 || value_type == VALUE_TYPE_F32 @@ -245,6 +246,25 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) param_count, param_types, result_count, result_types)) return false; break; + } + + case EXT_OP_BLOCK: + case EXT_OP_LOOP: + case EXT_OP_IF: + { + read_leb_uint32(frame_ip, frame_ip_end, type_index); + func_type = comp_ctx->comp_data->func_types[type_index]; + param_count = func_type->param_count; + param_types = func_type->types; + result_count = func_type->result_count; + result_types = func_type->types + param_count; + if (!aot_compile_op_block( + comp_ctx, func_ctx, &frame_ip, frame_ip_end, + (uint32)(LABEL_TYPE_BLOCK + opcode - EXT_OP_BLOCK), + param_count, param_types, result_count, result_types)) + return false; + break; + } case WASM_OP_ELSE: if (!aot_compile_op_else(comp_ctx, func_ctx, &frame_ip)) diff --git a/core/iwasm/compilation/aot_emit_const.c b/core/iwasm/compilation/aot_emit_const.c index 2508a36e..4b38aa96 100644 --- a/core/iwasm/compilation/aot_emit_const.c +++ b/core/iwasm/compilation/aot_emit_const.c @@ -4,7 +4,7 @@ */ #include "aot_emit_const.h" -#include "aot_intrinsic.h" +#include "../aot/aot_intrinsic.h" bool aot_compile_op_i32_const(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, diff --git a/core/iwasm/compilation/aot_emit_exception.c b/core/iwasm/compilation/aot_emit_exception.c index 3a16f7f8..d40ccc6a 100644 --- a/core/iwasm/compilation/aot_emit_exception.c +++ b/core/iwasm/compilation/aot_emit_exception.c @@ -4,6 +4,7 @@ */ #include "aot_emit_exception.h" +#include "../interpreter/wasm_runtime.h" #include "../aot/aot_runtime.h" bool @@ -57,7 +58,7 @@ aot_emit_exception(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } /* Create LLVM function with const function pointer */ if (!(func_const = - I64_CONST((uint64)(uintptr_t)aot_set_exception_with_id)) + I64_CONST((uint64)(uintptr_t)jit_set_exception_with_id)) || !(func = LLVMConstIntToPtr(func_const, func_ptr_type))) { aot_set_last_error("create LLVM value failed."); return false; diff --git a/core/iwasm/compilation/aot_emit_function.c b/core/iwasm/compilation/aot_emit_function.c index 1065b4c7..1bc4dbc2 100644 --- a/core/iwasm/compilation/aot_emit_function.c +++ b/core/iwasm/compilation/aot_emit_function.c @@ -162,7 +162,7 @@ call_aot_invoke_native_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } /* JIT mode, call the function directly */ - if (!(func = I64_CONST((uint64)(uintptr_t)aot_invoke_native)) + if (!(func = I64_CONST((uint64)(uintptr_t)llvm_jit_invoke_native)) || !(func = LLVMConstIntToPtr(func, func_ptr_type))) { aot_set_last_error("create LLVM value failed."); return false; @@ -407,7 +407,10 @@ call_aot_alloc_frame_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, param_types[1] = I32_TYPE; ret_type = INT8_TYPE; - GET_AOT_FUNCTION(aot_alloc_frame, 2); + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_alloc_frame, 2); + else + GET_AOT_FUNCTION(aot_alloc_frame, 2); param_values[0] = func_ctx->exec_env; param_values[1] = func_idx; @@ -461,7 +464,10 @@ call_aot_free_frame_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) param_types[0] = comp_ctx->exec_env_type; ret_type = INT8_TYPE; - GET_AOT_FUNCTION(aot_free_frame, 1); + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_free_frame, 1); + else + GET_AOT_FUNCTION(aot_free_frame, 1); param_values[0] = func_ctx->exec_env; @@ -557,7 +563,7 @@ check_app_addr_and_convert(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* JIT mode, call the function directly */ if (!(func = - I64_CONST((uint64)(uintptr_t)aot_check_app_addr_and_convert)) + I64_CONST((uint64)(uintptr_t)jit_check_app_addr_and_convert)) || !(func = LLVMConstIntToPtr(func, func_ptr_type))) { aot_set_last_error("create LLVM value failed."); return false; @@ -1047,7 +1053,7 @@ call_aot_call_indirect_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } /* JIT mode, call the function directly */ - if (!(func = I64_CONST((uint64)(uintptr_t)aot_call_indirect)) + if (!(func = I64_CONST((uint64)(uintptr_t)llvm_jit_call_indirect)) || !(func = LLVMConstIntToPtr(func, func_ptr_type))) { aot_set_last_error("create LLVM value failed."); return false; @@ -1265,7 +1271,7 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* load data as i32* */ if (!(offset = I32_CONST(get_tbl_inst_offset(comp_ctx, func_ctx, tbl_idx) - + offsetof(AOTTableInstance, data)))) { + + offsetof(AOTTableInstance, elems)))) { HANDLE_FAILURE("LLVMConstInt"); goto fail; } diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index 05e13b30..130f1830 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -718,7 +718,7 @@ aot_compile_op_memory_grow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) aot_set_last_error("llvm add pointer type failed."); return false; } - if (!(value = I64_CONST((uint64)(uintptr_t)aot_enlarge_memory)) + if (!(value = I64_CONST((uint64)(uintptr_t)wasm_enlarge_memory)) || !(func = LLVMConstIntToPtr(value, func_ptr_type))) { aot_set_last_error("create LLVM value failed."); return false; @@ -899,7 +899,10 @@ aot_compile_op_memory_init(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, param_types[4] = I32_TYPE; ret_type = INT8_TYPE; - GET_AOT_FUNCTION(aot_memory_init, 5); + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_memory_init, 5); + else + GET_AOT_FUNCTION(aot_memory_init, 5); /* Call function aot_memory_init() */ param_values[0] = func_ctx->aot_inst; @@ -955,7 +958,10 @@ aot_compile_op_data_drop(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, param_types[1] = I32_TYPE; ret_type = INT8_TYPE; - GET_AOT_FUNCTION(aot_data_drop, 2); + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_data_drop, 2); + else + GET_AOT_FUNCTION(aot_data_drop, 2); /* Call function aot_data_drop() */ param_values[0] = func_ctx->aot_inst; @@ -971,6 +977,14 @@ fail: return false; } +#if WASM_ENABLE_LAZY_JIT != 0 +static void * +jit_memmove(void *dest, const void *src, size_t n) +{ + return memmove(dest, src, n); +} +#endif + bool aot_compile_op_memory_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) { @@ -1025,7 +1039,7 @@ aot_compile_op_memory_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) return false; } #else - if (!(func = I64_CONST((uint64)(uintptr_t)aot_memmove)) + if (!(func = I64_CONST((uint64)(uintptr_t)jit_memmove)) || !(func = LLVMConstIntToPtr(func, func_ptr_type))) { aot_set_last_error("create LLVM value failed."); return false; @@ -1054,6 +1068,12 @@ fail: return false; } +static void * +jit_memset(void *s, int c, size_t n) +{ + return memset(s, c, n); +} + bool aot_compile_op_memory_fill(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) { @@ -1084,7 +1104,7 @@ aot_compile_op_memory_fill(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) } if (comp_ctx->is_jit_mode) { - if (!(func = I64_CONST((uint64)(uintptr_t)aot_memset)) + if (!(func = I64_CONST((uint64)(uintptr_t)jit_memset)) || !(func = LLVMConstIntToPtr(func, func_ptr_type))) { aot_set_last_error("create LLVM value failed."); return false; diff --git a/core/iwasm/compilation/aot_emit_table.c b/core/iwasm/compilation/aot_emit_table.c index 984ce8a0..9b18bd58 100644 --- a/core/iwasm/compilation/aot_emit_table.c +++ b/core/iwasm/compilation/aot_emit_table.c @@ -15,16 +15,25 @@ get_tbl_inst_offset(const AOTCompContext *comp_ctx, AOTImportTable *imp_tbls = comp_ctx->comp_data->import_tables; AOTTable *tbls = comp_ctx->comp_data->tables; - /* from the head of AOTModuleInstance */ - offset = - offsetof(AOTModuleInstance, global_table_data.bytes) - + (uint64)comp_ctx->comp_data->memory_count * sizeof(AOTMemoryInstance) - + comp_ctx->comp_data->global_data_size; + if (comp_ctx->is_jit_mode) { + offset = offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint64)comp_ctx->comp_data->memory_count + * sizeof(AOTMemoryInstance) + + comp_ctx->comp_data->global_data_size; + } + else { + offset = offsetof(AOTModuleInstance, global_table_data.bytes) + + (uint64)comp_ctx->comp_data->memory_count + * sizeof(AOTMemoryInstance) + + comp_ctx->comp_data->global_data_size; + } while (i < tbl_idx && i < comp_ctx->comp_data->import_table_count) { - offset += offsetof(AOTTableInstance, data); + offset += offsetof(AOTTableInstance, elems); /* avoid loading from current AOTTableInstance */ - offset += sizeof(uint32) * aot_get_imp_tbl_data_slots(imp_tbls + i); + offset += + sizeof(uint32) + * aot_get_imp_tbl_data_slots(imp_tbls + i, comp_ctx->is_jit_mode); ++i; } @@ -35,9 +44,10 @@ get_tbl_inst_offset(const AOTCompContext *comp_ctx, tbl_idx -= comp_ctx->comp_data->import_table_count; i -= comp_ctx->comp_data->import_table_count; while (i < tbl_idx && i < comp_ctx->comp_data->table_count) { - offset += offsetof(AOTTableInstance, data); + offset += offsetof(AOTTableInstance, elems); /* avoid loading from current AOTTableInstance */ - offset += sizeof(uint32) * aot_get_tbl_data_slots(tbls + i); + offset += sizeof(uint32) + * aot_get_tbl_data_slots(tbls + i, comp_ctx->is_jit_mode); ++i; } @@ -82,7 +92,10 @@ aot_compile_op_elem_drop(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, param_types[1] = I32_TYPE; ret_type = VOID_TYPE; - GET_AOT_FUNCTION(aot_drop_table_seg, 2); + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_drop_table_seg, 2); + else + GET_AOT_FUNCTION(aot_drop_table_seg, 2); param_values[0] = func_ctx->aot_inst; if (!(param_values[1] = I32_CONST(tbl_seg_idx))) { @@ -176,7 +189,7 @@ aot_compile_op_table_get(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* load data as i32* */ if (!(offset = I32_CONST(get_tbl_inst_offset(comp_ctx, func_ctx, tbl_idx) - + offsetof(AOTTableInstance, data)))) { + + offsetof(AOTTableInstance, elems)))) { HANDLE_FAILURE("LLVMConstInt"); goto fail; } @@ -230,7 +243,7 @@ aot_compile_op_table_set(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* load data as i32* */ if (!(offset = I32_CONST(get_tbl_inst_offset(comp_ctx, func_ctx, tbl_idx) - + offsetof(AOTTableInstance, data)))) { + + offsetof(AOTTableInstance, elems)))) { HANDLE_FAILURE("LLVMConstInt"); goto fail; } @@ -282,7 +295,10 @@ aot_compile_op_table_init(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, param_types[5] = I32_TYPE; ret_type = VOID_TYPE; - GET_AOT_FUNCTION(aot_table_init, 6); + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_table_init, 6); + else + GET_AOT_FUNCTION(aot_table_init, 6); param_values[0] = func_ctx->aot_inst; @@ -330,7 +346,10 @@ aot_compile_op_table_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, param_types[5] = I32_TYPE; ret_type = VOID_TYPE; - GET_AOT_FUNCTION(aot_table_copy, 6); + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_table_copy, 6); + else + GET_AOT_FUNCTION(aot_table_copy, 6); param_values[0] = func_ctx->aot_inst; @@ -414,7 +433,10 @@ aot_compile_op_table_grow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, param_types[3] = I32_TYPE; ret_type = I32_TYPE; - GET_AOT_FUNCTION(aot_table_grow, 4); + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_table_grow, 4); + else + GET_AOT_FUNCTION(aot_table_grow, 4); param_values[0] = func_ctx->aot_inst; @@ -455,7 +477,10 @@ aot_compile_op_table_fill(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, param_types[4] = I32_TYPE; ret_type = VOID_TYPE; - GET_AOT_FUNCTION(aot_table_fill, 5); + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_table_fill, 5); + else + GET_AOT_FUNCTION(aot_table_fill, 5); param_values[0] = func_ctx->aot_inst; diff --git a/core/iwasm/compilation/aot_emit_variable.c b/core/iwasm/compilation/aot_emit_variable.c index 4510a4c3..590638ce 100644 --- a/core/iwasm/compilation/aot_emit_variable.c +++ b/core/iwasm/compilation/aot_emit_variable.c @@ -114,14 +114,23 @@ compile_global(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, { AOTCompData *comp_data = comp_ctx->comp_data; uint32 import_global_count = comp_data->import_global_count; - uint32 global_base_offset = - offsetof(AOTModuleInstance, global_table_data.bytes) - + sizeof(AOTMemoryInstance) * comp_ctx->comp_data->memory_count; + uint32 global_base_offset; uint32 global_offset; uint8 global_type; LLVMValueRef offset, global_ptr, global, res; LLVMTypeRef ptr_type = NULL; + if (comp_ctx->is_jit_mode) { + global_base_offset = + offsetof(WASMModuleInstance, global_table_data.bytes) + + sizeof(WASMMemoryInstance) * comp_ctx->comp_data->memory_count; + } + else { + global_base_offset = + offsetof(AOTModuleInstance, global_table_data.bytes) + + sizeof(AOTMemoryInstance) * comp_ctx->comp_data->memory_count; + } + bh_assert(global_idx < import_global_count + comp_data->global_count); if (global_idx < import_global_count) { diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index 5f052bf2..aa65fcc5 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -264,7 +264,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return false; } /* memories[0]->memory_data */ - offset = I32_CONST(offsetof(AOTMemoryInstance, memory_data.ptr)); + offset = I32_CONST(offsetof(AOTMemoryInstance, memory_data)); if (!(func_ctx->mem_info[0].mem_base_addr = LLVMBuildInBoundsGEP2( comp_ctx->builder, INT8_TYPE, shared_mem_addr, &offset, 1, "mem_base_addr_offset"))) { @@ -292,15 +292,24 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, else #endif { - offset = I32_CONST(offsetof(AOTModuleInstance, global_table_data) - + offsetof(AOTMemoryInstance, memory_data.ptr)); + uint32 offset_of_global_table_data; + + if (comp_ctx->is_jit_mode) + offset_of_global_table_data = + offsetof(WASMModuleInstance, global_table_data); + else + offset_of_global_table_data = + offsetof(AOTModuleInstance, global_table_data); + + offset = I32_CONST(offset_of_global_table_data + + offsetof(AOTMemoryInstance, memory_data)); if (!(func_ctx->mem_info[0].mem_base_addr = LLVMBuildInBoundsGEP2( comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst, &offset, 1, "mem_base_addr_offset"))) { aot_set_last_error("llvm build in bounds gep failed"); return false; } - offset = I32_CONST(offsetof(AOTModuleInstance, global_table_data) + offset = I32_CONST(offset_of_global_table_data + offsetof(AOTMemoryInstance, cur_page_count)); if (!(func_ctx->mem_info[0].mem_cur_page_count_addr = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, @@ -309,7 +318,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, aot_set_last_error("llvm build in bounds gep failed"); return false; } - offset = I32_CONST(offsetof(AOTModuleInstance, global_table_data) + offset = I32_CONST(offset_of_global_table_data + offsetof(AOTMemoryInstance, memory_data_size)); if (!(func_ctx->mem_info[0].mem_data_size_addr = LLVMBuildInBoundsGEP2( comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst, &offset, 1, @@ -379,7 +388,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* Load memory bound check constants */ offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_1byte) - - offsetof(AOTMemoryInstance, memory_data.ptr)); + - offsetof(AOTMemoryInstance, memory_data)); if (!(func_ctx->mem_info[0].mem_bound_check_1byte = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, mem_info_base, &offset, 1, "bound_check_1byte_offset"))) { @@ -405,7 +414,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_2bytes) - - offsetof(AOTMemoryInstance, memory_data.ptr)); + - offsetof(AOTMemoryInstance, memory_data)); if (!(func_ctx->mem_info[0].mem_bound_check_2bytes = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, mem_info_base, &offset, 1, "bound_check_2bytes_offset"))) { @@ -431,7 +440,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_4bytes) - - offsetof(AOTMemoryInstance, memory_data.ptr)); + - offsetof(AOTMemoryInstance, memory_data)); if (!(func_ctx->mem_info[0].mem_bound_check_4bytes = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, mem_info_base, &offset, 1, "bound_check_4bytes_offset"))) { @@ -457,7 +466,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_8bytes) - - offsetof(AOTMemoryInstance, memory_data.ptr)); + - offsetof(AOTMemoryInstance, memory_data)); if (!(func_ctx->mem_info[0].mem_bound_check_8bytes = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, mem_info_base, &offset, 1, "bound_check_8bytes_offset"))) { @@ -483,7 +492,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_16bytes) - - offsetof(AOTMemoryInstance, memory_data.ptr)); + - offsetof(AOTMemoryInstance, memory_data)); if (!(func_ctx->mem_info[0].mem_bound_check_16bytes = LLVMBuildInBoundsGEP2( comp_ctx->builder, INT8_TYPE, mem_info_base, &offset, 1, "bound_check_16bytes_offset"))) { @@ -533,7 +542,7 @@ create_func_type_indexes(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) LLVMValueRef offset, func_type_indexes_ptr; LLVMTypeRef int32_ptr_type; - offset = I32_CONST(offsetof(AOTModuleInstance, func_type_indexes.ptr)); + offset = I32_CONST(offsetof(AOTModuleInstance, func_type_indexes)); func_type_indexes_ptr = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst, &offset, 1, "func_type_indexes_ptr"); diff --git a/core/iwasm/compilation/aot_llvm_extra.cpp b/core/iwasm/compilation/aot_llvm_extra.cpp index ed9fabd9..f0c205a8 100644 --- a/core/iwasm/compilation/aot_llvm_extra.cpp +++ b/core/iwasm/compilation/aot_llvm_extra.cpp @@ -338,8 +338,8 @@ aot_lookup_orcjit_func(LLVMOrcLLJITRef orc_lazyjit, void *module_inst, LLVMErrorRef error; LLVMOrcJITTargetAddress func_addr = 0; AOTModuleInstance *aot_inst = (AOTModuleInstance *)module_inst; - AOTModule *aot_module = (AOTModule *)aot_inst->aot_module.ptr; - void **func_ptrs = (void **)aot_inst->func_ptrs.ptr; + AOTModule *aot_module = (AOTModule *)aot_inst->module; + void **func_ptrs = aot_inst->func_ptrs; /** * No need to lock the func_ptr[func_idx] here as it is basic diff --git a/core/iwasm/fast-jit/fe/jit_emit_control.c b/core/iwasm/fast-jit/fe/jit_emit_control.c index eafce674..95e356c3 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_control.c +++ b/core/iwasm/fast-jit/fe/jit_emit_control.c @@ -1200,7 +1200,7 @@ jit_compile_op_return(JitCompContext *cc, uint8 **p_frame_ip) bool jit_compile_op_unreachable(JitCompContext *cc, uint8 **p_frame_ip) { - if (!jit_emit_exception(cc, JIT_EXCE_UNREACHABLE, JIT_OP_JMP, 0, NULL)) + if (!jit_emit_exception(cc, EXCE_UNREACHABLE, JIT_OP_JMP, 0, NULL)) return false; return handle_next_reachable_block(cc, p_frame_ip); diff --git a/core/iwasm/fast-jit/fe/jit_emit_conversion.c b/core/iwasm/fast-jit/fe/jit_emit_conversion.c index e753f929..8308a3ca 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_conversion.c +++ b/core/iwasm/fast-jit/fe/jit_emit_conversion.c @@ -201,19 +201,19 @@ jit_compile_check_value_range(JitCompContext *cc, JitReg value, JitReg min_fp, goto fail; GEN_INSN(CMP, cc->cmp_reg, nan_ret, NEW_CONST(I32, 1)); - if (!jit_emit_exception(cc, JIT_EXCE_INVALID_CONVERSION_TO_INTEGER, - JIT_OP_BEQ, cc->cmp_reg, NULL)) + if (!jit_emit_exception(cc, EXCE_INVALID_CONVERSION_TO_INTEGER, JIT_OP_BEQ, + cc->cmp_reg, NULL)) goto fail; /* If value is out of integer range, throw exception */ GEN_INSN(CMP, cc->cmp_reg, min_fp, value); - if (!jit_emit_exception(cc, JIT_EXCE_INTEGER_OVERFLOW, JIT_OP_BGES, - cc->cmp_reg, NULL)) + if (!jit_emit_exception(cc, EXCE_INTEGER_OVERFLOW, JIT_OP_BGES, cc->cmp_reg, + NULL)) goto fail; GEN_INSN(CMP, cc->cmp_reg, value, max_fp); - if (!jit_emit_exception(cc, JIT_EXCE_INTEGER_OVERFLOW, JIT_OP_BGES, - cc->cmp_reg, NULL)) + if (!jit_emit_exception(cc, EXCE_INTEGER_OVERFLOW, JIT_OP_BGES, cc->cmp_reg, + NULL)) goto fail; return true; diff --git a/core/iwasm/fast-jit/fe/jit_emit_exception.c b/core/iwasm/fast-jit/fe/jit_emit_exception.c index 5e227685..2addb5cd 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_exception.c +++ b/core/iwasm/fast-jit/fe/jit_emit_exception.c @@ -14,7 +14,7 @@ jit_emit_exception(JitCompContext *cc, int32 exception_id, uint8 jit_opcode, JitIncomingInsn *incoming_insn; JitReg else_label; - bh_assert(exception_id < JIT_EXCE_NUM); + bh_assert(exception_id < EXCE_NUM); if (jit_opcode >= JIT_OP_BEQ && jit_opcode <= JIT_OP_BLEU) { bh_assert(cond_br_if == cc->cmp_reg); diff --git a/core/iwasm/fast-jit/fe/jit_emit_function.c b/core/iwasm/fast-jit/fe/jit_emit_function.c index 257615f0..7f83231c 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_function.c +++ b/core/iwasm/fast-jit/fe/jit_emit_function.c @@ -9,10 +9,6 @@ #include "../jit_codegen.h" #include "../../interpreter/wasm_runtime.h" -extern bool -jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, - WASMInterpFrame *prev_frame); - /* Prepare parameters for the function to call */ static bool pre_call(JitCompContext *cc, const WASMType *func_type) @@ -187,8 +183,8 @@ jit_compile_op_call(JitCompContext *cc, uint32 func_idx, bool tail_call) arg_regs[1] = NEW_CONST(I32, func_idx); arg_regs[2] = cc->fp_reg; - if (!jit_emit_callnative(cc, jit_invoke_native, native_ret, arg_regs, - 3)) { + if (!jit_emit_callnative(cc, fast_jit_invoke_native, native_ret, + arg_regs, 3)) { return false; } /* Convert bool to uint32 */ @@ -196,7 +192,7 @@ jit_compile_op_call(JitCompContext *cc, uint32 func_idx, bool tail_call) /* Check whether there is exception thrown */ GEN_INSN(CMP, cc->cmp_reg, native_ret, NEW_CONST(I32, 0)); - if (!jit_emit_exception(cc, JIT_EXCE_ALREADY_THROWN, JIT_OP_BEQ, + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BEQ, cc->cmp_reg, NULL)) { return false; } @@ -355,7 +351,8 @@ jit_compile_op_call_indirect(JitCompContext *cc, uint32 type_idx, arg_regs[4] = NEW_CONST(I32, func_type->param_cell_num); arg_regs[5] = argv; - if (!jit_emit_callnative(cc, jit_call_indirect, native_ret, arg_regs, 6)) { + if (!jit_emit_callnative(cc, fast_jit_call_indirect, native_ret, arg_regs, + 6)) { return false; } /* Convert bool to uint32 */ @@ -363,8 +360,8 @@ jit_compile_op_call_indirect(JitCompContext *cc, uint32 type_idx, /* Check whether there is exception thrown */ GEN_INSN(CMP, cc->cmp_reg, native_ret, NEW_CONST(I32, 0)); - if (!jit_emit_exception(cc, JIT_EXCE_ALREADY_THROWN, JIT_OP_BEQ, - cc->cmp_reg, NULL)) { + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BEQ, cc->cmp_reg, + NULL)) { return false; } diff --git a/core/iwasm/fast-jit/fe/jit_emit_memory.c b/core/iwasm/fast-jit/fe/jit_emit_memory.c index 6daf5c0d..19687039 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_memory.c +++ b/core/iwasm/fast-jit/fe/jit_emit_memory.c @@ -78,8 +78,8 @@ check_and_seek_on_64bit_platform(JitCompContext *cc, JitReg addr, JitReg offset, #ifndef OS_ENABLE_HW_BOUND_CHECK /* if (offset1 > memory_boundary) goto EXCEPTION */ GEN_INSN(CMP, cc->cmp_reg, offset1, memory_boundary); - if (!jit_emit_exception(cc, JIT_EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, - JIT_OP_BGTU, cc->cmp_reg, NULL)) { + if (!jit_emit_exception(cc, EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, JIT_OP_BGTU, + cc->cmp_reg, NULL)) { goto fail; } #endif @@ -103,16 +103,16 @@ check_and_seek_on_32bit_platform(JitCompContext *cc, JitReg addr, JitReg offset, /* if (offset1 < addr) goto EXCEPTION */ GEN_INSN(CMP, cc->cmp_reg, offset1, addr); - if (!jit_emit_exception(cc, JIT_EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, - JIT_OP_BLTU, cc->cmp_reg, NULL)) { + if (!jit_emit_exception(cc, EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, JIT_OP_BLTU, + cc->cmp_reg, NULL)) { goto fail; } #ifndef OS_ENABLE_HW_BOUND_CHECK /* if (offset1 > memory_boundary) goto EXCEPTION */ GEN_INSN(CMP, cc->cmp_reg, offset1, memory_boundary); - if (!jit_emit_exception(cc, JIT_EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, - JIT_OP_BGTU, cc->cmp_reg, NULL)) { + if (!jit_emit_exception(cc, EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, JIT_OP_BGTU, + cc->cmp_reg, NULL)) { goto fail; } #endif @@ -144,7 +144,7 @@ check_and_seek(JitCompContext *cc, JitReg addr, uint32 offset, uint32 bytes) GEN_INSN(LDI32, cur_mem_page_count, memory_inst, NEW_CONST(I32, offsetof(WASMMemoryInstance, cur_page_count))); GEN_INSN(CMP, cc->cmp_reg, cur_mem_page_count, NEW_CONST(I32, 0)); - if (!jit_emit_exception(cc, JIT_EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, + if (!jit_emit_exception(cc, EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, JIT_OP_BEQ, cc->cmp_reg, NULL)) { goto fail; } @@ -604,8 +604,8 @@ jit_compile_op_memory_init(JitCompContext *cc, uint32 mem_idx, uint32 seg_idx) goto fail; GEN_INSN(CMP, cc->cmp_reg, res, NEW_CONST(I32, 0)); - if (!jit_emit_exception(cc, JIT_EXCE_ALREADY_THROWN, JIT_OP_BLTS, - cc->cmp_reg, NULL)) + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BLTS, cc->cmp_reg, + NULL)) goto fail; return true; @@ -687,8 +687,8 @@ jit_compile_op_memory_copy(JitCompContext *cc, uint32 src_mem_idx, goto fail; GEN_INSN(CMP, cc->cmp_reg, res, NEW_CONST(I32, 0)); - if (!jit_emit_exception(cc, JIT_EXCE_ALREADY_THROWN, JIT_OP_BLTS, - cc->cmp_reg, NULL)) + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BLTS, cc->cmp_reg, + NULL)) goto fail; return true; @@ -741,8 +741,8 @@ jit_compile_op_memory_fill(JitCompContext *cc, uint32 mem_idx) goto fail; GEN_INSN(CMP, cc->cmp_reg, res, NEW_CONST(I32, 0)); - if (!jit_emit_exception(cc, JIT_EXCE_ALREADY_THROWN, JIT_OP_BLTS, - cc->cmp_reg, NULL)) + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BLTS, cc->cmp_reg, + NULL)) goto fail; return true; diff --git a/core/iwasm/fast-jit/fe/jit_emit_numberic.c b/core/iwasm/fast-jit/fe/jit_emit_numberic.c index 3b2f21af..04c8f780 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_numberic.c +++ b/core/iwasm/fast-jit/fe/jit_emit_numberic.c @@ -665,7 +665,7 @@ compile_int_div(JitCompContext *cc, IntArithmetic arith_op, bool is_i32, case 0: { /* Directly throw exception if divided by zero */ - if (!(jit_emit_exception(cc, JIT_EXCE_INTEGER_DIVIDE_BY_ZERO, + if (!(jit_emit_exception(cc, EXCE_INTEGER_DIVIDE_BY_ZERO, JIT_OP_JMP, 0, NULL))) goto fail; @@ -699,7 +699,7 @@ compile_int_div(JitCompContext *cc, IntArithmetic arith_op, bool is_i32, /* Throw integer overflow exception if left is INT32_MIN or INT64_MIN */ - if (!(jit_emit_exception(cc, JIT_EXCE_INTEGER_OVERFLOW, + if (!(jit_emit_exception(cc, EXCE_INTEGER_OVERFLOW, JIT_OP_BEQ, cc->cmp_reg, NULL))) goto fail; @@ -739,8 +739,8 @@ compile_int_div(JitCompContext *cc, IntArithmetic arith_op, bool is_i32, GEN_INSN(CMP, cc->cmp_reg, right, is_i32 ? NEW_CONST(I32, 0) : NEW_CONST(I64, 0)); /* Throw integer divided by zero exception if right is zero */ - if (!(jit_emit_exception(cc, JIT_EXCE_INTEGER_DIVIDE_BY_ZERO, - JIT_OP_BEQ, cc->cmp_reg, NULL))) + if (!(jit_emit_exception(cc, EXCE_INTEGER_DIVIDE_BY_ZERO, JIT_OP_BEQ, + cc->cmp_reg, NULL))) goto fail; switch (arith_op) { @@ -760,8 +760,8 @@ compile_int_div(JitCompContext *cc, IntArithmetic arith_op, bool is_i32, GEN_INSN(CMP, cc->cmp_reg, cmp1, NEW_CONST(I32, 1)); /* Throw integer overflow exception if left is INT32_MIN or INT64_MIN, and right is -1 */ - if (!(jit_emit_exception(cc, JIT_EXCE_INTEGER_OVERFLOW, - JIT_OP_BEQ, cc->cmp_reg, NULL))) + if (!(jit_emit_exception(cc, EXCE_INTEGER_OVERFLOW, JIT_OP_BEQ, + cc->cmp_reg, NULL))) goto fail; /* Build default div and rem */ diff --git a/core/iwasm/fast-jit/fe/jit_emit_table.c b/core/iwasm/fast-jit/fe/jit_emit_table.c index 57017113..0b24a663 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_table.c +++ b/core/iwasm/fast-jit/fe/jit_emit_table.c @@ -37,8 +37,8 @@ jit_compile_op_table_get(JitCompContext *cc, uint32 tbl_idx) /* if (elem_idx >= tbl_sz) goto exception; */ tbl_sz = get_table_cur_size_reg(cc->jit_frame, tbl_idx); GEN_INSN(CMP, cc->cmp_reg, elem_idx, tbl_sz); - if (!jit_emit_exception(cc, JIT_EXCE_OUT_OF_BOUNDS_TABLE_ACCESS, - JIT_OP_BGEU, cc->cmp_reg, NULL)) + if (!jit_emit_exception(cc, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS, JIT_OP_BGEU, + cc->cmp_reg, NULL)) goto fail; elem_idx_long = jit_cc_new_reg_I64(cc); @@ -68,8 +68,8 @@ jit_compile_op_table_set(JitCompContext *cc, uint32 tbl_idx) /* if (elem_idx >= tbl_sz) goto exception; */ tbl_sz = get_table_cur_size_reg(cc->jit_frame, tbl_idx); GEN_INSN(CMP, cc->cmp_reg, elem_idx, tbl_sz); - if (!jit_emit_exception(cc, JIT_EXCE_OUT_OF_BOUNDS_TABLE_ACCESS, - JIT_OP_BGEU, cc->cmp_reg, NULL)) + if (!jit_emit_exception(cc, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS, JIT_OP_BGEU, + cc->cmp_reg, NULL)) goto fail; elem_idx_long = jit_cc_new_reg_I64(cc); @@ -105,7 +105,7 @@ wasm_init_table(WASMModuleInstance *inst, uint32 tbl_idx, uint32 elem_idx, if (src > elem_len || elem_len - src < len) goto out_of_bounds; - bh_memcpy_s((uint8 *)(tbl) + offsetof(WASMTableInstance, base_addr) + bh_memcpy_s((uint8 *)tbl + offsetof(WASMTableInstance, elems) + dst * sizeof(uint32), (uint32)((tbl_sz - dst) * sizeof(uint32)), elem->func_indexes + src, (uint32)(len * sizeof(uint32))); @@ -140,8 +140,8 @@ jit_compile_op_table_init(JitCompContext *cc, uint32 tbl_idx, goto fail; GEN_INSN(CMP, cc->cmp_reg, res, NEW_CONST(I32, 0)); - if (!jit_emit_exception(cc, JIT_EXCE_ALREADY_THROWN, JIT_OP_BLTS, - cc->cmp_reg, NULL)) + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BLTS, cc->cmp_reg, + NULL)) goto fail; return true; @@ -167,10 +167,10 @@ wasm_copy_table(WASMModuleInstance *inst, uint32 src_tbl_idx, if (dst_offset > dst_tbl_sz || dst_tbl_sz - dst_offset < len) goto out_of_bounds; - bh_memmove_s((uint8 *)(dst_tbl) + offsetof(WASMTableInstance, base_addr) + bh_memmove_s((uint8 *)dst_tbl + offsetof(WASMTableInstance, elems) + dst_offset * sizeof(uint32), (uint32)((dst_tbl_sz - dst_offset) * sizeof(uint32)), - (uint8 *)(src_tbl) + offsetof(WASMTableInstance, base_addr) + (uint8 *)src_tbl + offsetof(WASMTableInstance, elems) + src_offset * sizeof(uint32), (uint32)(len * sizeof(uint32))); @@ -204,8 +204,8 @@ jit_compile_op_table_copy(JitCompContext *cc, uint32 src_tbl_idx, goto fail; GEN_INSN(CMP, cc->cmp_reg, res, NEW_CONST(I32, 0)); - if (!jit_emit_exception(cc, JIT_EXCE_ALREADY_THROWN, JIT_OP_BLTS, - cc->cmp_reg, NULL)) + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BLTS, cc->cmp_reg, + NULL)) goto fail; return true; @@ -276,7 +276,7 @@ wasm_fill_table(WASMModuleInstance *inst, uint32 tbl_idx, uint32 dst, goto out_of_bounds; for (; len != 0; dst++, len--) { - ((uint32 *)(tbl->base_addr))[dst] = val; + tbl->elems[dst] = val; } return 0; @@ -307,8 +307,8 @@ jit_compile_op_table_fill(JitCompContext *cc, uint32 tbl_idx) goto fail; GEN_INSN(CMP, cc->cmp_reg, res, NEW_CONST(I32, 0)); - if (!jit_emit_exception(cc, JIT_EXCE_ALREADY_THROWN, JIT_OP_BLTS, - cc->cmp_reg, NULL)) + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BLTS, cc->cmp_reg, + NULL)) goto fail; return true; diff --git a/core/iwasm/fast-jit/fe/jit_emit_variable.c b/core/iwasm/fast-jit/fe/jit_emit_variable.c index e018d220..ba039d38 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_variable.c +++ b/core/iwasm/fast-jit/fe/jit_emit_variable.c @@ -277,11 +277,11 @@ jit_compile_op_set_global(JitCompContext *cc, uint32 global_idx, JitReg aux_stack_bottom = get_aux_stack_bottom_reg(cc->jit_frame); GEN_INSN(CMP, cc->cmp_reg, value, aux_stack_bound); - if (!(jit_emit_exception(cc, JIT_EXCE_AUX_STACK_OVERFLOW, + if (!(jit_emit_exception(cc, EXCE_AUX_STACK_OVERFLOW, JIT_OP_BLEU, cc->cmp_reg, NULL))) goto fail; GEN_INSN(CMP, cc->cmp_reg, value, aux_stack_bottom); - if (!(jit_emit_exception(cc, JIT_EXCE_AUX_STACK_UNDERFLOW, + if (!(jit_emit_exception(cc, EXCE_AUX_STACK_UNDERFLOW, JIT_OP_BGTU, cc->cmp_reg, NULL))) goto fail; } diff --git a/core/iwasm/fast-jit/jit_frontend.c b/core/iwasm/fast-jit/jit_frontend.c index 90e20da3..dc65123f 100644 --- a/core/iwasm/fast-jit/jit_frontend.c +++ b/core/iwasm/fast-jit/jit_frontend.c @@ -18,31 +18,9 @@ #include "fe/jit_emit_variable.h" #include "../interpreter/wasm_interp.h" #include "../interpreter/wasm_opcode.h" +#include "../interpreter/wasm_runtime.h" #include "../common/wasm_exec_env.h" -/* clang-format off */ -static const char *jit_exception_msgs[] = { - "unreachable", /* JIT_EXCE_UNREACHABLE */ - "allocate memory failed", /* JIT_EXCE_OUT_OF_MEMORY */ - "out of bounds memory access", /* JIT_EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS */ - "integer overflow", /* JIT_EXCE_INTEGER_OVERFLOW */ - "integer divide by zero", /* JIT_EXCE_INTEGER_DIVIDE_BY_ZERO */ - "invalid conversion to integer", /* JIT_EXCE_INVALID_CONVERSION_TO_INTEGER */ - "indirect call type mismatch", /* JIT_EXCE_INVALID_FUNCTION_TYPE_INDEX */ - "invalid function index", /* JIT_EXCE_INVALID_FUNCTION_INDEX */ - "undefined element", /* JIT_EXCE_UNDEFINED_ELEMENT */ - "uninitialized element", /* JIT_EXCE_UNINITIALIZED_ELEMENT */ - "failed to call unlinked import function", /* JIT_EXCE_CALL_UNLINKED_IMPORT_FUNC */ - "native stack overflow", /* JIT_EXCE_NATIVE_STACK_OVERFLOW */ - "unaligned atomic", /* JIT_EXCE_UNALIGNED_ATOMIC */ - "wasm auxiliary stack overflow", /* JIT_EXCE_AUX_STACK_OVERFLOW */ - "wasm auxiliary stack underflow", /* JIT_EXCE_AUX_STACK_UNDERFLOW */ - "out of bounds table access", /* JIT_EXCE_OUT_OF_BOUNDS_TABLE_ACCESS */ - "wasm operand stack overflow", /* JIT_EXCE_OPERAND_STACK_OVERFLOW */ - "", /* JIT_EXCE_ALREADY_THROWN */ -}; -/* clang-format on */ - JitReg get_module_inst_reg(JitFrame *frame) { @@ -378,7 +356,7 @@ get_table_data_reg(JitFrame *frame, uint32 tbl_idx) frame->table_regs[tbl_idx].table_data = cc->table_regs[tbl_idx].table_data; GEN_INSN(ADD, frame->table_regs[tbl_idx].table_data, table_reg, - NEW_CONST(I64, offsetof(WASMTableInstance, base_addr))); + NEW_CONST(I64, offsetof(WASMTableInstance, elems))); } return frame->table_regs[tbl_idx].table_data; } @@ -586,15 +564,6 @@ gen_commit_sp_ip(JitFrame *frame) #endif } -static void -jit_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id) -{ - if (id < JIT_EXCE_NUM) - wasm_set_exception(module_inst, jit_exception_msgs[id]); - else - wasm_set_exception(module_inst, "unknown exception"); -} - static bool create_fixed_virtual_regs(JitCompContext *cc) { @@ -681,7 +650,7 @@ form_and_translate_func(JitCompContext *cc) jit_basic_block_append_insn(jit_cc_entry_basic_block(cc), insn); /* Patch INSNs jumping to exception basic blocks. */ - for (i = 0; i < JIT_EXCE_NUM; i++) { + for (i = 0; i < EXCE_NUM; i++) { incoming_insn = cc->incoming_insns_for_exec_bbs[i]; if (incoming_insn) { if (!(cc->exce_basic_blocks[i] = jit_cc_new_basic_block(cc, 0))) { @@ -703,7 +672,7 @@ form_and_translate_func(JitCompContext *cc) incoming_insn = incoming_insn_next; } cc->cur_basic_block = cc->exce_basic_blocks[i]; - if (i != JIT_EXCE_ALREADY_THROWN) { + if (i != EXCE_ALREADY_THROWN) { JitReg module_inst_reg = jit_cc_new_reg_ptr(cc); GEN_INSN(LDPTR, module_inst_reg, cc->exec_env_reg, NEW_CONST(I32, offsetof(WASMExecEnv, module_inst))); @@ -850,7 +819,7 @@ init_func_translation(JitCompContext *cc) GEN_INSN(ADD, frame_boundary, top, NEW_CONST(PTR, frame_size + outs_size)); /* if frame_boundary > top_boundary, throw stack overflow exception */ GEN_INSN(CMP, cc->cmp_reg, frame_boundary, top_boundary); - if (!jit_emit_exception(cc, JIT_EXCE_OPERAND_STACK_OVERFLOW, JIT_OP_BGTU, + if (!jit_emit_exception(cc, EXCE_OPERAND_STACK_OVERFLOW, JIT_OP_BGTU, cc->cmp_reg, NULL)) { return NULL; } diff --git a/core/iwasm/fast-jit/jit_frontend.h b/core/iwasm/fast-jit/jit_frontend.h index eeecf48f..725d930a 100644 --- a/core/iwasm/fast-jit/jit_frontend.h +++ b/core/iwasm/fast-jit/jit_frontend.h @@ -104,28 +104,6 @@ typedef enum FloatArithmetic { FLOAT_MAX, } FloatArithmetic; -typedef enum JitExceptionID { - JIT_EXCE_UNREACHABLE = 0, - JIT_EXCE_OUT_OF_MEMORY, - JIT_EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, - JIT_EXCE_INTEGER_OVERFLOW, - JIT_EXCE_INTEGER_DIVIDE_BY_ZERO, - JIT_EXCE_INVALID_CONVERSION_TO_INTEGER, - JIT_EXCE_INVALID_FUNCTION_TYPE_INDEX, - JIT_EXCE_INVALID_FUNCTION_INDEX, - JIT_EXCE_UNDEFINED_ELEMENT, - JIT_EXCE_UNINITIALIZED_ELEMENT, - JIT_EXCE_CALL_UNLINKED_IMPORT_FUNC, - JIT_EXCE_NATIVE_STACK_OVERFLOW, - JIT_EXCE_UNALIGNED_ATOMIC, - JIT_EXCE_AUX_STACK_OVERFLOW, - JIT_EXCE_AUX_STACK_UNDERFLOW, - JIT_EXCE_OUT_OF_BOUNDS_TABLE_ACCESS, - JIT_EXCE_OPERAND_STACK_OVERFLOW, - JIT_EXCE_ALREADY_THROWN, - JIT_EXCE_NUM, -} JitExceptionID; - /** * Translate instructions in a function. The translated block must * end with a branch instruction whose targets are offsets relating to diff --git a/core/iwasm/fast-jit/jit_ir.c b/core/iwasm/fast-jit/jit_ir.c index 6efc2eff..9fc8d5cc 100644 --- a/core/iwasm/fast-jit/jit_ir.c +++ b/core/iwasm/fast-jit/jit_ir.c @@ -382,6 +382,7 @@ jit_cc_init(JitCompContext *cc, unsigned htab_size) blocks respectively. */ if (!(entry_block = jit_cc_new_basic_block(cc, 0))) goto fail; + if (!(exit_block = jit_cc_new_basic_block(cc, 0))) { jit_basic_block_delete(entry_block); goto fail; @@ -395,11 +396,11 @@ jit_cc_init(JitCompContext *cc, unsigned htab_size) && jit_reg_no(cc->exit_label) == 1); if (!(cc->exce_basic_blocks = - jit_calloc(sizeof(JitBasicBlock *) * JIT_EXCE_NUM))) + jit_calloc(sizeof(JitBasicBlock *) * EXCE_NUM))) goto fail; if (!(cc->incoming_insns_for_exec_bbs = - jit_calloc(sizeof(JitIncomingInsnList) * JIT_EXCE_NUM))) + jit_calloc(sizeof(JitIncomingInsnList) * EXCE_NUM))) goto fail; cc->hreg_info = jit_codegen_get_hreg_info(); @@ -465,7 +466,7 @@ jit_cc_destroy(JitCompContext *cc) jit_free(cc->exce_basic_blocks); if (cc->incoming_insns_for_exec_bbs) { - for (i = 0; i < JIT_EXCE_NUM; i++) { + for (i = 0; i < EXCE_NUM; i++) { incoming_insn = cc->incoming_insns_for_exec_bbs[i]; while (incoming_insn) { incoming_insn_next = incoming_insn->next; diff --git a/core/iwasm/include/wasm_c_api.h b/core/iwasm/include/wasm_c_api.h index 1c9d6a18..84832364 100644 --- a/core/iwasm/include/wasm_c_api.h +++ b/core/iwasm/include/wasm_c_api.h @@ -461,7 +461,7 @@ WASM_API_EXTERN bool wasm_module_validate(wasm_store_t*, const wasm_byte_vec_t* WASM_API_EXTERN void wasm_module_imports(const wasm_module_t*, own wasm_importtype_vec_t* out); WASM_API_EXTERN void wasm_module_exports(const wasm_module_t*, own wasm_exporttype_vec_t* out); -WASM_API_EXTERN void wasm_module_serialize(const wasm_module_t*, own wasm_byte_vec_t* out); +WASM_API_EXTERN void wasm_module_serialize(wasm_module_t*, own wasm_byte_vec_t* out); WASM_API_EXTERN own wasm_module_t* wasm_module_deserialize(wasm_store_t*, const wasm_byte_vec_t*); diff --git a/core/iwasm/interpreter/wasm.h b/core/iwasm/interpreter/wasm.h index 44b3973d..ad0972b7 100644 --- a/core/iwasm/interpreter/wasm.h +++ b/core/iwasm/interpreter/wasm.h @@ -263,6 +263,9 @@ struct WASMFunction { #if WASM_ENABLE_FAST_JIT != 0 void *fast_jit_jitted_code; #endif +#if WASM_ENABLE_JIT != 0 + void *llvm_jit_func_ptr; +#endif }; struct WASMGlobal { @@ -363,6 +366,11 @@ typedef struct WASMCustomSection { } WASMCustomSection; #endif +#if WASM_ENABLE_JIT != 0 +struct AOTCompData; +struct AOTCompContext; +#endif + struct WASMModule { /* Module type, for module loaded from WASM bytecode binary, this field is Wasm_Module_Bytecode; @@ -407,6 +415,9 @@ struct WASMModule { WASMDataSeg **data_segments; uint32 start_function; + /* total global variable size */ + uint32 global_data_size; + /* the index of auxiliary __data_end global, -1 means unexported */ uint32 aux_data_end_global_index; @@ -493,6 +504,12 @@ struct WASMModule { /* point to JITed functions */ void **fast_jit_func_ptrs; #endif + +#if WASM_ENABLE_JIT != 0 + struct AOTCompData *comp_data; + struct AOTCompContext *comp_ctx; + void **func_ptrs; +#endif }; typedef struct BlockType { diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index b7edcf7c..30c40baf 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -641,28 +641,28 @@ trunc_f64_to_int(WASMModuleInstance *module, uint32 *frame_sp, float64 src_min, CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = (uint32)(*(uint8 *)maddr); \ *(uint8 *)maddr = (uint8)(readv op sval); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ else if (opcode == WASM_OP_ATOMIC_RMW_I32_##OP_NAME##16_U) { \ CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = (uint32)LOAD_U16(maddr); \ STORE_U16(maddr, (uint16)(readv op sval)); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ else { \ CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = LOAD_I32(maddr); \ STORE_U32(maddr, readv op sval); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ PUSH_I32(readv); \ break; \ @@ -681,39 +681,39 @@ trunc_f64_to_int(WASMModuleInstance *module, uint32 *frame_sp, float64 src_min, CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = (uint64)(*(uint8 *)maddr); \ *(uint8 *)maddr = (uint8)(readv op sval); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##16_U) { \ CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = (uint64)LOAD_U16(maddr); \ STORE_U16(maddr, (uint16)(readv op sval)); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##32_U) { \ CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = (uint64)LOAD_U32(maddr); \ STORE_U32(maddr, (uint32)(readv op sval)); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ else { \ uint64 op_result; \ CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = (uint64)LOAD_I64(maddr); \ op_result = readv op sval; \ STORE_I64(maddr, op_result); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ PUSH_I64(readv); \ break; \ @@ -850,7 +850,7 @@ wasm_interp_call_func_native(WASMModuleInstance *module_inst, wasm_exec_env_set_cur_frame(exec_env, frame); - cur_func_index = (uint32)(cur_func - module_inst->functions); + cur_func_index = (uint32)(cur_func - module_inst->e->functions); bh_assert(cur_func_index < module_inst->module->import_function_count); native_func_pointer = module_inst->import_func_ptrs[cur_func_index]; @@ -904,12 +904,12 @@ wasm_interp_call_func_native(WASMModuleInstance *module_inst, #if WASM_ENABLE_FAST_JIT != 0 bool -jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, - WASMInterpFrame *prev_frame) +fast_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, + WASMInterpFrame *prev_frame) { WASMModuleInstance *module_inst = (WASMModuleInstance *)exec_env->module_inst; - WASMFunctionInstance *cur_func = module_inst->functions + func_idx; + WASMFunctionInstance *cur_func = module_inst->e->functions + func_idx; wasm_interp_call_func_native(module_inst, exec_env, cur_func, prev_frame); return wasm_get_exception(module_inst) ? false : true; @@ -1081,7 +1081,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, WASMFunctionInstance *cur_func, WASMInterpFrame *prev_frame) { - WASMMemoryInstance *memory = module->default_memory; + WASMMemoryInstance *memory = wasm_get_default_memory(module); uint8 *global_data = module->global_data; #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ @@ -1091,7 +1091,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, memory ? num_bytes_per_page * memory->cur_page_count : 0; #endif WASMType **wasm_types = module->module->types; - WASMGlobalInstance *globals = module->globals, *global; + WASMGlobalInstance *globals = module->e->globals, *global; uint8 opcode_IMPDEP = WASM_OP_IMPDEP; WASMInterpFrame *frame = NULL; /* Points to this special opcode so as to jump to the @@ -1355,13 +1355,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #endif read_leb_uint32(frame_ip, frame_ip_end, fidx); #if WASM_ENABLE_MULTI_MODULE != 0 - if (fidx >= module->function_count) { + if (fidx >= module->e->function_count) { wasm_set_exception(module, "unknown function"); goto got_exception; } #endif - cur_func = module->functions + fidx; + cur_func = module->e->functions + fidx; goto call_func_from_interp; } @@ -1373,12 +1373,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #endif read_leb_uint32(frame_ip, frame_ip_end, fidx); #if WASM_ENABLE_MULTI_MODULE != 0 - if (fidx >= module->function_count) { + if (fidx >= module->e->function_count) { wasm_set_exception(module, "unknown function"); goto got_exception; } #endif - cur_func = module->functions + fidx; + cur_func = module->e->functions + fidx; goto call_func_from_return_call; } @@ -1420,8 +1420,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; } - fidx = ((uint32 *)tbl_inst->base_addr)[val]; - if (fidx == (uint32)-1) { + fidx = tbl_inst->elems[val]; + if (fidx == NULL_REF) { wasm_set_exception(module, "uninitialized element"); goto got_exception; } @@ -1431,13 +1431,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, * another module. In that case, we don't validate * the elem value while loading */ - if (fidx >= module->function_count) { + if (fidx >= module->e->function_count) { wasm_set_exception(module, "unknown function"); goto got_exception; } /* always call module own functions */ - cur_func = module->functions + fidx; + cur_func = module->e->functions + fidx; if (cur_func->is_import_func) cur_func_type = cur_func->u.func_import->func_type; @@ -1531,7 +1531,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; } - PUSH_I32(((uint32 *)tbl_inst->base_addr)[elem_idx]); + PUSH_I32(tbl_inst->elems[elem_idx]); HANDLE_OP_END(); } @@ -1552,7 +1552,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; } - ((uint32 *)(tbl_inst->base_addr))[elem_idx] = elem_val; + tbl_inst->elems[elem_idx] = elem_val; HANDLE_OP_END(); } @@ -1700,7 +1700,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_GET_GLOBAL) { read_leb_uint32(frame_ip, frame_ip_end, global_idx); - bh_assert(global_idx < module->global_count); + bh_assert(global_idx < module->e->global_count); global = globals + global_idx; global_addr = get_global_addr(global_data, global); PUSH_I32(*(uint32 *)global_addr); @@ -1710,7 +1710,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_GET_GLOBAL_64) { read_leb_uint32(frame_ip, frame_ip_end, global_idx); - bh_assert(global_idx < module->global_count); + bh_assert(global_idx < module->e->global_count); global = globals + global_idx; global_addr = get_global_addr(global_data, global); PUSH_I64(GET_I64_FROM_ADDR((uint32 *)global_addr)); @@ -1720,7 +1720,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_SET_GLOBAL) { read_leb_uint32(frame_ip, frame_ip_end, global_idx); - bh_assert(global_idx < module->global_count); + bh_assert(global_idx < module->e->global_count); global = globals + global_idx; global_addr = get_global_addr(global_data, global); *(int32 *)global_addr = POP_I32(); @@ -1732,7 +1732,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, uint32 aux_stack_top; read_leb_uint32(frame_ip, frame_ip_end, global_idx); - bh_assert(global_idx < module->global_count); + bh_assert(global_idx < module->e->global_count); global = globals + global_idx; global_addr = get_global_addr(global_data, global); aux_stack_top = *(uint32 *)(frame_sp - 1); @@ -1751,8 +1751,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (module->module->aux_stack_top_global_index != (uint32)-1) { uint32 aux_stack_used = module->module->aux_stack_bottom - *(uint32 *)global_addr; - if (aux_stack_used > module->max_aux_stack_used) - module->max_aux_stack_used = aux_stack_used; + if (aux_stack_used > module->e->max_aux_stack_used) + module->e->max_aux_stack_used = aux_stack_used; } #endif HANDLE_OP_END(); @@ -1761,7 +1761,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_SET_GLOBAL_64) { read_leb_uint32(frame_ip, frame_ip_end, global_idx); - bh_assert(global_idx < module->global_count); + bh_assert(global_idx < module->e->global_count); global = globals + global_idx; global_addr = get_global_addr(global_data, global); PUT_I64_TO_ADDR((uint32 *)global_addr, POP_I64()); @@ -2039,8 +2039,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, else { /* success, return previous page count */ PUSH_I32(prev_page_count); - /* update memory instance ptr and memory size */ - memory = module->default_memory; + /* update memory size, no need to update memory ptr as + it isn't changed in wasm_enlarge_memory */ #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ || WASM_ENABLE_BULK_MEMORY != 0 @@ -3195,8 +3195,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, } bh_memcpy_s( - (uint8 *)(tbl_inst) - + offsetof(WASMTableInstance, base_addr) + (uint8 *)tbl_inst + + offsetof(WASMTableInstance, elems) + d * sizeof(uint32), (uint32)((tbl_inst->cur_size - d) * sizeof(uint32)), module->module->table_segments[elem_idx] @@ -3246,16 +3246,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, /* if s >= d, copy from front to back */ /* if s < d, copy from back to front */ /* merge all together */ - bh_memmove_s( - (uint8 *)(dst_tbl_inst) - + offsetof(WASMTableInstance, base_addr) - + d * sizeof(uint32), - (uint32)((dst_tbl_inst->cur_size - d) - * sizeof(uint32)), - (uint8 *)(src_tbl_inst) - + offsetof(WASMTableInstance, base_addr) - + s * sizeof(uint32), - (uint32)(n * sizeof(uint32))); + bh_memmove_s((uint8 *)dst_tbl_inst + + offsetof(WASMTableInstance, elems) + + d * sizeof(uint32), + (uint32)((dst_tbl_inst->cur_size - d) + * sizeof(uint32)), + (uint8 *)src_tbl_inst + + offsetof(WASMTableInstance, elems) + + s * sizeof(uint32), + (uint32)(n * sizeof(uint32))); break; } case WASM_OP_TABLE_GROW: @@ -3319,7 +3318,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, } for (; n != 0; i++, n--) { - ((uint32 *)(tbl_inst->base_addr))[i] = fill_val; + tbl_inst->elems[i] = fill_val; } break; @@ -3420,23 +3419,23 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (opcode == WASM_OP_ATOMIC_I32_LOAD8_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint32)(*(uint8 *)maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_I32_LOAD16_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint32)LOAD_U16(maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = LOAD_I32(maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } PUSH_I32(readv); @@ -3455,30 +3454,30 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (opcode == WASM_OP_ATOMIC_I64_LOAD8_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)(*(uint8 *)maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_I64_LOAD16_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)LOAD_U16(maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_I64_LOAD32_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)LOAD_U32(maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = LOAD_I64(maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } PUSH_I64(readv); @@ -3497,23 +3496,23 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (opcode == WASM_OP_ATOMIC_I32_STORE8) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); *(uint8 *)maddr = (uint8)sval; - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_I32_STORE16) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); STORE_U16(maddr, (uint16)sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); STORE_U32(maddr, frame_sp[1]); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } break; } @@ -3531,31 +3530,31 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (opcode == WASM_OP_ATOMIC_I64_STORE8) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); *(uint8 *)maddr = (uint8)sval; - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_I64_STORE16) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); STORE_U16(maddr, (uint16)sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_I64_STORE32) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); STORE_U32(maddr, (uint32)sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); PUT_I64_TO_ADDR((uint32 *)maddr, GET_I64_FROM_ADDR(frame_sp + 1)); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } break; } @@ -3575,32 +3574,32 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, CHECK_ATOMIC_MEMORY_ACCESS(); expect = (uint8)expect; - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint32)(*(uint8 *)maddr); if (readv == expect) *(uint8 *)maddr = (uint8)(sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); expect = (uint16)expect; - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint32)LOAD_U16(maddr); if (readv == expect) STORE_U16(maddr, (uint16)(sval)); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = LOAD_I32(maddr); if (readv == expect) STORE_U32(maddr, sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } PUSH_I32(readv); break; @@ -3621,44 +3620,44 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, CHECK_ATOMIC_MEMORY_ACCESS(); expect = (uint8)expect; - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)(*(uint8 *)maddr); if (readv == expect) *(uint8 *)maddr = (uint8)(sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); expect = (uint16)expect; - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)LOAD_U16(maddr); if (readv == expect) STORE_U16(maddr, (uint16)(sval)); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); expect = (uint32)expect; - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)LOAD_U32(maddr); if (readv == expect) STORE_U32(maddr, (uint32)(sval)); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)LOAD_I64(maddr); if (readv == expect) { STORE_I64(maddr, sval); } - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } PUSH_I64(readv); break; @@ -3794,8 +3793,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, cur_func = frame->function; UPDATE_ALL_FROM_FRAME(); - /* update memory instance ptr and memory size */ - memory = module->default_memory; + /* update memory size, no need to update memory ptr as + it isn't changed in wasm_enlarge_memory */ #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ || WASM_ENABLE_BULK_MEMORY != 0 @@ -3905,6 +3904,191 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #endif } +#if WASM_ENABLE_FAST_JIT != 0 +static void +fast_jit_call_func_bytecode(WASMExecEnv *exec_env, + WASMFunctionInstance *function, + WASMInterpFrame *frame) +{ + JitGlobals *jit_globals = jit_compiler_get_jit_globals(); + JitInterpSwitchInfo info; + WASMType *func_type = function->u.func->func_type; + uint8 type = func_type->result_count + ? func_type->types[func_type->param_count] + : VALUE_TYPE_VOID; + +#if WASM_ENABLE_REF_TYPES != 0 + if (type == VALUE_TYPE_EXTERNREF || type == VALUE_TYPE_FUNCREF) + type = VALUE_TYPE_I32; +#endif + + info.out.ret.last_return_type = type; + info.frame = frame; + frame->jitted_return_addr = + (uint8 *)jit_globals->return_to_interp_from_jitted; + jit_interp_switch_to_jitted(exec_env, &info, + function->u.func->fast_jit_jitted_code); + if (func_type->result_count) { + switch (type) { + case VALUE_TYPE_I32: + *(frame->sp - function->ret_cell_num) = info.out.ret.ival[0]; + break; + case VALUE_TYPE_I64: + *(frame->sp - function->ret_cell_num) = info.out.ret.ival[0]; + *(frame->sp - function->ret_cell_num + 1) = + info.out.ret.ival[1]; + break; + case VALUE_TYPE_F32: + *(frame->sp - function->ret_cell_num) = info.out.ret.fval[0]; + break; + case VALUE_TYPE_F64: + *(frame->sp - function->ret_cell_num) = info.out.ret.fval[0]; + *(frame->sp - function->ret_cell_num + 1) = + info.out.ret.fval[1]; + break; + default: + bh_assert(0); + break; + } + } +} +#endif + +#if WASM_ENABLE_JIT != 0 +static bool +clear_wasi_proc_exit_exception(WASMModuleInstance *module_inst) +{ +#if WASM_ENABLE_LIBC_WASI != 0 + const char *exception = wasm_get_exception(module_inst); + if (exception && !strcmp(exception, "Exception: wasi proc exit")) { + /* The "wasi proc exit" exception is thrown by native lib to + let wasm app exit, which is a normal behavior, we clear + the exception here. */ + wasm_set_exception(module_inst, NULL); + return true; + } + return false; +#else + return false; +#endif +} + +static bool +llvm_jit_call_func_bytecode(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, + WASMFunctionInstance *function, uint32 argc, + uint32 argv[]) +{ + WASMType *func_type = function->u.func->func_type; + uint32 result_count = func_type->result_count; + uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0; + bool ret; + +#if (WASM_ENABLE_DUMP_CALL_STACK != 0) || (WASM_ENABLE_PERF_PROFILING != 0) + if (!llvm_jit_alloc_frame(exec_env, function - module_inst->e->functions)) { + wasm_set_exception(module_inst, "wasm operand stack overflow"); + } +#endif + + if (ext_ret_count > 0) { + uint32 cell_num = 0, i; + uint8 *ext_ret_types = func_type->types + func_type->param_count + 1; + uint32 argv1_buf[32], *argv1 = argv1_buf, *ext_rets = NULL; + uint32 *argv_ret = argv; + uint32 ext_ret_cell = wasm_get_cell_num(ext_ret_types, ext_ret_count); + uint64 size; + + /* Allocate memory all arguments */ + size = + sizeof(uint32) * (uint64)argc /* original arguments */ + + sizeof(void *) + * (uint64)ext_ret_count /* extra result values' addr */ + + sizeof(uint32) * (uint64)ext_ret_cell; /* extra result values */ + if (size > sizeof(argv1_buf)) { + if (size > UINT32_MAX + || !(argv1 = wasm_runtime_malloc((uint32)size))) { + wasm_set_exception(module_inst, "allocate memory failed"); + return false; + } + } + + /* Copy original arguments */ + bh_memcpy_s(argv1, (uint32)size, argv, sizeof(uint32) * argc); + + /* Get the extra result value's address */ + ext_rets = + argv1 + argc + sizeof(void *) / sizeof(uint32) * ext_ret_count; + + /* Append each extra result value's address to original arguments */ + for (i = 0; i < ext_ret_count; i++) { + *(uintptr_t *)(argv1 + argc + sizeof(void *) / sizeof(uint32) * i) = + (uintptr_t)(ext_rets + cell_num); + cell_num += wasm_value_type_cell_num(ext_ret_types[i]); + } + + ret = wasm_runtime_invoke_native( + exec_env, function->u.func->llvm_jit_func_ptr, func_type, NULL, + NULL, argv1, argc, argv); + + if (!ret || wasm_get_exception(module_inst)) { + if (clear_wasi_proc_exit_exception(module_inst)) + ret = true; + else + ret = false; + } + + if (!ret) { + if (argv1 != argv1_buf) + wasm_runtime_free(argv1); + return ret; + } + + /* Get extra result values */ + switch (func_type->types[func_type->param_count]) { + case VALUE_TYPE_I32: + case VALUE_TYPE_F32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: +#endif + argv_ret++; + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + argv_ret += 2; + break; +#if WASM_ENABLE_SIMD != 0 + case VALUE_TYPE_V128: + argv_ret += 4; + break; +#endif + default: + bh_assert(0); + break; + } + + ext_rets = + argv1 + argc + sizeof(void *) / sizeof(uint32) * ext_ret_count; + bh_memcpy_s(argv_ret, sizeof(uint32) * cell_num, ext_rets, + sizeof(uint32) * cell_num); + + if (argv1 != argv1_buf) + wasm_runtime_free(argv1); + return true; + } + else { + ret = wasm_runtime_invoke_native( + exec_env, function->u.func->llvm_jit_func_ptr, func_type, NULL, + NULL, argv, argc, argv); + + if (clear_wasi_proc_exit_exception(module_inst)) + ret = true; + + return ret && !wasm_get_exception(module_inst) ? true : false; + } +} +#endif + void wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env, WASMFunctionInstance *function, uint32 argc, @@ -3912,14 +4096,14 @@ wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env, { WASMRuntimeFrame *prev_frame = wasm_exec_env_get_cur_frame(exec_env); WASMInterpFrame *frame, *outs_area; - /* Allocate sufficient cells for all kinds of return values. */ unsigned all_cell_num = - function->ret_cell_num > 2 ? function->ret_cell_num : 2, - i; + function->ret_cell_num > 2 ? function->ret_cell_num : 2; /* This frame won't be used by JITed code, so only allocate interp frame here. */ unsigned frame_size = wasm_interp_interp_frame_size(all_cell_num); + unsigned i; + bool copy_argv_from_frame = true; if (argc < function->param_cell_num) { char buf[128]; @@ -3969,62 +4153,26 @@ wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env, } } else { -#if WASM_ENABLE_FAST_JIT == 0 - wasm_interp_call_func_bytecode(module_inst, exec_env, function, frame); +#if WASM_ENABLE_JIT != 0 + llvm_jit_call_func_bytecode(module_inst, exec_env, function, argc, + argv); + /* For llvm jit, the results have been stored in argv, + no need to copy them from stack frame again */ + copy_argv_from_frame = false; +#elif WASM_ENABLE_FAST_JIT != 0 + fast_jit_call_func_bytecode(exec_env, function, frame); #else - JitGlobals *jit_globals = jit_compiler_get_jit_globals(); - JitInterpSwitchInfo info; - WASMType *func_type = function->u.func->func_type; - uint8 type = func_type->result_count - ? func_type->types[func_type->param_count] - : VALUE_TYPE_VOID; - -#if WASM_ENABLE_REF_TYPES != 0 - if (type == VALUE_TYPE_EXTERNREF || type == VALUE_TYPE_FUNCREF) - type = VALUE_TYPE_I32; + wasm_interp_call_func_bytecode(module_inst, exec_env, function, frame); #endif - - info.out.ret.last_return_type = type; - info.frame = frame; - frame->jitted_return_addr = - (uint8 *)jit_globals->return_to_interp_from_jitted; - jit_interp_switch_to_jitted(exec_env, &info, - function->u.func->fast_jit_jitted_code); - if (func_type->result_count) { - switch (type) { - case VALUE_TYPE_I32: - *(frame->sp - function->ret_cell_num) = - info.out.ret.ival[0]; - break; - case VALUE_TYPE_I64: - *(frame->sp - function->ret_cell_num) = - info.out.ret.ival[0]; - *(frame->sp - function->ret_cell_num + 1) = - info.out.ret.ival[1]; - break; - case VALUE_TYPE_F32: - *(frame->sp - function->ret_cell_num) = - info.out.ret.fval[0]; - break; - case VALUE_TYPE_F64: - *(frame->sp - function->ret_cell_num) = - info.out.ret.fval[0]; - *(frame->sp - function->ret_cell_num + 1) = - info.out.ret.fval[1]; - break; - default: - bh_assert(0); - break; - } - } (void)wasm_interp_call_func_bytecode; -#endif } /* Output the return value to the caller */ if (!wasm_get_exception(module_inst)) { - for (i = 0; i < function->ret_cell_num; i++) { - argv[i] = *(frame->sp + i - function->ret_cell_num); + if (copy_argv_from_frame) { + for (i = 0; i < function->ret_cell_num; i++) { + argv[i] = *(frame->sp + i - function->ret_cell_num); + } } } else { diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 2f9c654a..d2440d18 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -443,28 +443,28 @@ LOAD_PTR(void *addr) CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(1); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = (uint32)(*(uint8 *)maddr); \ *(uint8 *)maddr = (uint8)(readv op sval); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ else if (opcode == WASM_OP_ATOMIC_RMW_I32_##OP_NAME##16_U) { \ CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(2); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = (uint32)LOAD_U16(maddr); \ STORE_U16(maddr, (uint16)(readv op sval)); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ else { \ CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(4); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = LOAD_I32(maddr); \ STORE_U32(maddr, readv op sval); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ PUSH_I32(readv); \ break; \ @@ -483,39 +483,39 @@ LOAD_PTR(void *addr) CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(1); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = (uint64)(*(uint8 *)maddr); \ *(uint8 *)maddr = (uint8)(readv op sval); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##16_U) { \ CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(2); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = (uint64)LOAD_U16(maddr); \ STORE_U16(maddr, (uint16)(readv op sval)); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##32_U) { \ CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(4); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = (uint64)LOAD_U32(maddr); \ STORE_U32(maddr, (uint32)(readv op sval)); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ else { \ uint64 op_result; \ CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(8); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = (uint64)LOAD_I64(maddr); \ op_result = readv op sval; \ STORE_I64(maddr, op_result); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ PUSH_I64(readv); \ break; \ @@ -913,7 +913,7 @@ wasm_interp_call_func_native(WASMModuleInstance *module_inst, wasm_exec_env_set_cur_frame(exec_env, frame); - cur_func_index = (uint32)(cur_func - module_inst->functions); + cur_func_index = (uint32)(cur_func - module_inst->e->functions); bh_assert(cur_func_index < module_inst->module->import_function_count); native_func_pointer = module_inst->import_func_ptrs[cur_func_index]; @@ -1138,7 +1138,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, WASMFunctionInstance *cur_func, WASMInterpFrame *prev_frame) { - WASMMemoryInstance *memory = module->default_memory; + WASMMemoryInstance *memory = wasm_get_default_memory(module); #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ || WASM_ENABLE_BULK_MEMORY != 0 @@ -1147,7 +1147,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, memory ? num_bytes_per_page * memory->cur_page_count : 0; #endif uint8 *global_data = module->global_data; - WASMGlobalInstance *globals = module->globals, *global; + WASMGlobalInstance *globals = module->e ? module->e->globals : NULL; + WASMGlobalInstance *global; uint8 opcode_IMPDEP = WASM_OP_IMPDEP; WASMInterpFrame *frame = NULL; /* Points to this special opcode so as to jump to the @@ -1345,8 +1346,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; } - fidx = ((uint32 *)tbl_inst->base_addr)[val]; - if (fidx == (uint32)-1) { + fidx = tbl_inst->elems[val]; + if (fidx == NULL_REF) { wasm_set_exception(module, "uninitialized element"); goto got_exception; } @@ -1356,13 +1357,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, * another module. in that case, we don't validate * the elem value while loading */ - if (fidx >= module->function_count) { + if (fidx >= module->e->function_count) { wasm_set_exception(module, "unknown function"); goto got_exception; } /* always call module own functions */ - cur_func = module->functions + fidx; + cur_func = module->e->functions + fidx; if (cur_func->is_import_func) cur_func_type = cur_func->u.func_import->func_type; @@ -1437,7 +1438,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; } - PUSH_I32(((uint32 *)tbl_inst->base_addr)[elem_idx]); + PUSH_I32(tbl_inst->elems[elem_idx]); HANDLE_OP_END(); } @@ -1458,7 +1459,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; } - ((uint32 *)tbl_inst->base_addr)[elem_idx] = elem_val; + tbl_inst->elems[elem_idx] = elem_val; HANDLE_OP_END(); } @@ -1522,7 +1523,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_GET_GLOBAL) { global_idx = read_uint32(frame_ip); - bh_assert(global_idx < module->global_count); + bh_assert(global_idx < module->e->global_count); global = globals + global_idx; global_addr = get_global_addr(global_data, global); addr_ret = GET_OFFSET(); @@ -1533,7 +1534,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_GET_GLOBAL_64) { global_idx = read_uint32(frame_ip); - bh_assert(global_idx < module->global_count); + bh_assert(global_idx < module->e->global_count); global = globals + global_idx; global_addr = get_global_addr(global_data, global); addr_ret = GET_OFFSET(); @@ -1545,7 +1546,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_SET_GLOBAL) { global_idx = read_uint32(frame_ip); - bh_assert(global_idx < module->global_count); + bh_assert(global_idx < module->e->global_count); global = globals + global_idx; global_addr = get_global_addr(global_data, global); addr1 = GET_OFFSET(); @@ -1558,7 +1559,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, uint32 aux_stack_top; global_idx = read_uint32(frame_ip); - bh_assert(global_idx < module->global_count); + bh_assert(global_idx < module->e->global_count); global = globals + global_idx; global_addr = get_global_addr(global_data, global); aux_stack_top = frame_lp[GET_OFFSET()]; @@ -1576,8 +1577,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (module->module->aux_stack_top_global_index != (uint32)-1) { uint32 aux_stack_used = module->module->aux_stack_bottom - *(uint32 *)global_addr; - if (aux_stack_used > module->max_aux_stack_used) - module->max_aux_stack_used = aux_stack_used; + if (aux_stack_used > module->e->max_aux_stack_used) + module->e->max_aux_stack_used = aux_stack_used; } #endif HANDLE_OP_END(); @@ -1586,7 +1587,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_SET_GLOBAL_64) { global_idx = read_uint32(frame_ip); - bh_assert(global_idx < module->global_count); + bh_assert(global_idx < module->e->global_count); global = globals + global_idx; global_addr = get_global_addr(global_data, global); addr1 = GET_OFFSET(); @@ -1860,8 +1861,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, else { /* success, return previous page count */ frame_lp[addr_ret] = prev_page_count; - /* update memory instance ptr and memory size */ - memory = module->default_memory; + /* update memory size, no need to update memory ptr as + it isn't changed in wasm_enlarge_memory */ #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ || WASM_ENABLE_BULK_MEMORY != 0 @@ -3093,7 +3094,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, bh_memcpy_s( (uint8 *)tbl_inst - + offsetof(WASMTableInstance, base_addr) + + offsetof(WASMTableInstance, elems) + d * sizeof(uint32), (uint32)((tbl_inst->cur_size - d) * sizeof(uint32)), module->module->table_segments[elem_idx] @@ -3141,16 +3142,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, /* if s >= d, copy from front to back */ /* if s < d, copy from back to front */ /* merge all together */ - bh_memmove_s( - (uint8 *)dst_tbl_inst - + offsetof(WASMTableInstance, base_addr) - + d * sizeof(uint32), - (uint32)((dst_tbl_inst->cur_size - d) - * sizeof(uint32)), - (uint8 *)src_tbl_inst - + offsetof(WASMTableInstance, base_addr) - + s * sizeof(uint32), - (uint32)(n * sizeof(uint32))); + bh_memmove_s((uint8 *)dst_tbl_inst + + offsetof(WASMTableInstance, elems) + + d * sizeof(uint32), + (uint32)((dst_tbl_inst->cur_size - d) + * sizeof(uint32)), + (uint8 *)src_tbl_inst + + offsetof(WASMTableInstance, elems) + + s * sizeof(uint32), + (uint32)(n * sizeof(uint32))); break; } case WASM_OP_TABLE_GROW: @@ -3211,7 +3211,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, } for (; n != 0; i++, n--) { - ((uint32 *)(tbl_inst->base_addr))[i] = fill_val; + tbl_inst->elems[i] = fill_val; } break; @@ -3305,23 +3305,23 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (opcode == WASM_OP_ATOMIC_I32_LOAD8_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); CHECK_ATOMIC_MEMORY_ACCESS(1); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint32)(*(uint8 *)maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_I32_LOAD16_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); CHECK_ATOMIC_MEMORY_ACCESS(2); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint32)LOAD_U16(maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); CHECK_ATOMIC_MEMORY_ACCESS(4); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = LOAD_I32(maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } PUSH_I32(readv); @@ -3340,30 +3340,30 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (opcode == WASM_OP_ATOMIC_I64_LOAD8_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); CHECK_ATOMIC_MEMORY_ACCESS(1); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)(*(uint8 *)maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_I64_LOAD16_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); CHECK_ATOMIC_MEMORY_ACCESS(2); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)LOAD_U16(maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_I64_LOAD32_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); CHECK_ATOMIC_MEMORY_ACCESS(4); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)LOAD_U32(maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); CHECK_ATOMIC_MEMORY_ACCESS(8); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = LOAD_I64(maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } PUSH_I64(readv); @@ -3381,23 +3381,23 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (opcode == WASM_OP_ATOMIC_I32_STORE8) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); CHECK_ATOMIC_MEMORY_ACCESS(1); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); *(uint8 *)maddr = (uint8)sval; - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_I32_STORE16) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); CHECK_ATOMIC_MEMORY_ACCESS(2); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); STORE_U16(maddr, (uint16)sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); CHECK_ATOMIC_MEMORY_ACCESS(4); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); STORE_U32(maddr, sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } break; } @@ -3415,30 +3415,30 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (opcode == WASM_OP_ATOMIC_I64_STORE8) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); CHECK_ATOMIC_MEMORY_ACCESS(1); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); *(uint8 *)maddr = (uint8)sval; - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_I64_STORE16) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); CHECK_ATOMIC_MEMORY_ACCESS(2); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); STORE_U16(maddr, (uint16)sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_I64_STORE32) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); CHECK_ATOMIC_MEMORY_ACCESS(4); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); STORE_U32(maddr, (uint32)sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); CHECK_ATOMIC_MEMORY_ACCESS(8); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); STORE_I64(maddr, sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } break; } @@ -3458,32 +3458,32 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, CHECK_ATOMIC_MEMORY_ACCESS(1); expect = (uint8)expect; - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint32)(*(uint8 *)maddr); if (readv == expect) *(uint8 *)maddr = (uint8)(sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); CHECK_ATOMIC_MEMORY_ACCESS(2); expect = (uint16)expect; - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint32)LOAD_U16(maddr); if (readv == expect) STORE_U16(maddr, (uint16)(sval)); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); CHECK_ATOMIC_MEMORY_ACCESS(4); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = LOAD_I32(maddr); if (readv == expect) STORE_U32(maddr, sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } PUSH_I32(readv); break; @@ -3504,44 +3504,44 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, CHECK_ATOMIC_MEMORY_ACCESS(1); expect = (uint8)expect; - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)(*(uint8 *)maddr); if (readv == expect) *(uint8 *)maddr = (uint8)(sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); CHECK_ATOMIC_MEMORY_ACCESS(2); expect = (uint16)expect; - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)LOAD_U16(maddr); if (readv == expect) STORE_U16(maddr, (uint16)(sval)); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); CHECK_ATOMIC_MEMORY_ACCESS(4); expect = (uint32)expect; - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)LOAD_U32(maddr); if (readv == expect) STORE_U32(maddr, (uint32)(sval)); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); CHECK_ATOMIC_MEMORY_ACCESS(8); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)LOAD_I64(maddr); if (readv == expect) { STORE_I64(maddr, sval); } - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } PUSH_I64(readv); break; @@ -3575,12 +3575,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #endif fidx = read_uint32(frame_ip); #if WASM_ENABLE_MULTI_MODULE != 0 - if (fidx >= module->function_count) { + if (fidx >= module->e->function_count) { wasm_set_exception(module, "unknown function"); goto got_exception; } #endif - cur_func = module->functions + fidx; + cur_func = module->e->functions + fidx; goto call_func_from_interp; } @@ -3592,12 +3592,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #endif fidx = read_uint32(frame_ip); #if WASM_ENABLE_MULTI_MODULE != 0 - if (fidx >= module->function_count) { + if (fidx >= module->e->function_count) { wasm_set_exception(module, "unknown function"); goto got_exception; } #endif - cur_func = module->functions + fidx; + cur_func = module->e->functions + fidx; goto call_func_from_return_call; } #endif /* WASM_ENABLE_TAIL_CALL */ @@ -3782,8 +3782,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, cur_func = frame->function; UPDATE_ALL_FROM_FRAME(); - /* update memory instance ptr and memory size */ - memory = module->default_memory; + /* update memory size, no need to update memory ptr as + it isn't changed in wasm_enlarge_memory */ #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ || WASM_ENABLE_BULK_MEMORY != 0 diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index e5941b7b..6ec7caab 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -18,6 +18,9 @@ #include "../fast-jit/jit_compiler.h" #include "../fast-jit/jit_codecache.h" #endif +#if WASM_ENABLE_JIT != 0 +#include "../compilation/aot_llvm.h" +#endif /* Read a value of given type from the address pointed to by the given pointer and increase the pointer to the position just after the @@ -2923,7 +2926,6 @@ fail: return false; } -#if WASM_ENABLE_FAST_JIT != 0 static void calculate_global_data_offset(WASMModule *module) { @@ -2933,17 +2935,127 @@ calculate_global_data_offset(WASMModule *module) for (i = 0; i < module->import_global_count; i++) { WASMGlobalImport *import_global = &((module->import_globals + i)->u.global); +#if WASM_ENABLE_FAST_JIT != 0 import_global->data_offset = data_offset; +#endif data_offset += wasm_value_type_size(import_global->type); } for (i = 0; i < module->global_count; i++) { WASMGlobal *global = module->globals + i; +#if WASM_ENABLE_FAST_JIT != 0 global->data_offset = data_offset; +#endif data_offset += wasm_value_type_size(global->type); } + + module->global_data_size = data_offset; } + +#if WASM_ENABLE_JIT != 0 +static bool +compile_llvm_jit_functions(WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + AOTCompOption option = { 0 }; + char func_name[32], *aot_last_error; + uint64 size; + uint32 i; + + size = sizeof(void *) * (uint64)module->function_count; + if (size > 0 + && !(module->func_ptrs = + loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + + module->comp_data = aot_create_comp_data(module); + if (!module->comp_data) { + aot_last_error = aot_get_last_error(); + bh_assert(aot_last_error != NULL); + set_error_buf(error_buf, error_buf_size, aot_last_error); + return false; + } + + option.is_jit_mode = true; + option.opt_level = 3; + option.size_level = 3; +#if WASM_ENABLE_BULK_MEMORY != 0 + option.enable_bulk_memory = true; #endif +#if WASM_ENABLE_THREAD_MGR != 0 + option.enable_thread_mgr = true; +#endif +#if WASM_ENABLE_TAIL_CALL != 0 + option.enable_tail_call = true; +#endif +#if WASM_ENABLE_SIMD != 0 + option.enable_simd = true; +#endif +#if WASM_ENABLE_REF_TYPES != 0 + option.enable_ref_types = true; +#endif + option.enable_aux_stack_check = true; +#if (WASM_ENABLE_PERF_PROFILING != 0) || (WASM_ENABLE_DUMP_CALL_STACK != 0) + option.enable_aux_stack_frame = true; +#endif + + module->comp_ctx = aot_create_comp_context(module->comp_data, &option); + if (!module->comp_ctx) { + aot_last_error = aot_get_last_error(); + bh_assert(aot_last_error != NULL); + set_error_buf(error_buf, error_buf_size, aot_last_error); + return false; + } + + if (!aot_compile_wasm(module->comp_ctx)) { + aot_last_error = aot_get_last_error(); + bh_assert(aot_last_error != NULL); + set_error_buf(error_buf, error_buf_size, aot_last_error); + return false; + } + +#if WASM_ENABLE_LAZY_JIT != 0 + for (i = 0; i < module->comp_data->func_count; i++) { + LLVMErrorRef error; + LLVMOrcJITTargetAddress func_addr = 0; + + snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i); + if ((error = LLVMOrcLLJITLookup(module->comp_ctx->orc_lazyjit, + &func_addr, func_name))) { + char *err_msg = LLVMGetErrorMessage(error); + set_error_buf_v(error_buf, error_buf_size, + "failed to compile orc jit function: %s", err_msg); + LLVMDisposeErrorMessage(err_msg); + return false; + } + /** + * No need to lock the func_ptr[func_idx] here as it is basic + * data type, the load/store for it can be finished by one cpu + * instruction, and there can be only one cpu instruction + * loading/storing at the same time. + */ + module->func_ptrs[i] = (void *)func_addr; + module->functions[i]->llvm_jit_func_ptr = (void *)func_addr; + } +#else + /* Resolve function addresses */ + bh_assert(module->comp_ctx->exec_engine); + for (i = 0; i < module->comp_data->func_count; i++) { + snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i); + if (!(module->func_ptrs[i] = (void *)LLVMGetFunctionAddress( + module->comp_ctx->exec_engine, func_name))) { + set_error_buf(error_buf, error_buf_size, + "failed to compile llvm mc jit function"); + return false; + } + module->functions[i]->llvm_jit_func_ptr = module->func_ptrs[i]; + } +#endif /* end of WASM_ENABLE_LAZY_JIT != 0 */ + + return true; +} +#endif /* end of WASM_ENABLE_JIT != 0 */ static bool wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func, @@ -3352,9 +3464,9 @@ load_from_sections(WASMModule *module, WASMSection *sections, #endif } -#if WASM_ENABLE_FAST_JIT != 0 calculate_global_data_offset(module); +#if WASM_ENABLE_FAST_JIT != 0 if (module->function_count && !(module->fast_jit_func_ptrs = loader_malloc(sizeof(void *) * module->function_count, @@ -3367,6 +3479,12 @@ load_from_sections(WASMModule *module, WASMSection *sections, } #endif +#if WASM_ENABLE_JIT != 0 + if (!compile_llvm_jit_functions(module, error_buf, error_buf_size)) { + return false; + } +#endif /* end of WASM_ENABLE_JIT != 0 */ + #if WASM_ENABLE_MEMORY_TRACING != 0 wasm_runtime_dump_module_mem_consumption((WASMModuleCommon *)module); #endif @@ -3900,6 +4018,15 @@ wasm_loader_unload(WASMModule *module) } #endif +#if WASM_ENABLE_JIT != 0 + if (module->func_ptrs) + wasm_runtime_free(module->func_ptrs); + if (module->comp_ctx) + aot_destroy_comp_context(module->comp_ctx); + if (module->comp_data) + aot_destroy_comp_data(module->comp_data); +#endif + wasm_runtime_free(module); } @@ -6756,8 +6883,7 @@ re_scan: } block_type.is_value_type = false; block_type.u.type = module->types[type_index]; -#if WASM_ENABLE_FAST_INTERP == 0 && WASM_ENABLE_WAMR_COMPILER == 0 \ - && WASM_ENABLE_JIT == 0 +#if WASM_ENABLE_FAST_INTERP == 0 /* If block use type index as block type, change the opcode * to new extended opcode so that interpreter can resolve * the block quickly. diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index a2f58956..1773e623 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -15,6 +15,9 @@ #include "../fast-jit/jit_compiler.h" #include "../fast-jit/jit_codecache.h" #endif +#if WASM_ENABLE_JIT != 0 +#include "../compilation/aot_llvm.h" +#endif /* Read a value of given type from the address pointed to by the given pointer and increase the pointer to the position just after the @@ -961,6 +964,11 @@ load_function_section(const uint8 *buf, const uint8 *buf_end, read_leb_uint32(p, p_end, type_index); bh_assert(type_index < module->type_count); +#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) + type_index = wasm_get_smallest_type_idx( + module->types, module->type_count, type_index); +#endif + read_leb_uint32(p_code, buf_code_end, code_size); bh_assert(code_size > 0 && p_code + code_size <= buf_code_end); @@ -1749,6 +1757,137 @@ load_user_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, return true; } +static void +calculate_global_data_offset(WASMModule *module) +{ + uint32 i, data_offset; + + data_offset = 0; + for (i = 0; i < module->import_global_count; i++) { + WASMGlobalImport *import_global = + &((module->import_globals + i)->u.global); +#if WASM_ENABLE_FAST_JIT != 0 + import_global->data_offset = data_offset; +#endif + data_offset += wasm_value_type_size(import_global->type); + } + + for (i = 0; i < module->global_count; i++) { + WASMGlobal *global = module->globals + i; +#if WASM_ENABLE_FAST_JIT != 0 + global->data_offset = data_offset; +#endif + data_offset += wasm_value_type_size(global->type); + } + + module->global_data_size = data_offset; +} + +#if WASM_ENABLE_JIT != 0 +static bool +compile_llvm_jit_functions(WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + AOTCompOption option = { 0 }; + char func_name[32], *aot_last_error; + uint64 size; + uint32 i; + + size = sizeof(void *) * (uint64)module->function_count; + if (size > 0 + && !(module->func_ptrs = + loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + + module->comp_data = aot_create_comp_data(module); + if (!module->comp_data) { + aot_last_error = aot_get_last_error(); + bh_assert(aot_last_error != NULL); + set_error_buf(error_buf, error_buf_size, aot_last_error); + return false; + } + + option.is_jit_mode = true; + option.opt_level = 3; + option.size_level = 3; +#if WASM_ENABLE_BULK_MEMORY != 0 + option.enable_bulk_memory = true; +#endif +#if WASM_ENABLE_THREAD_MGR != 0 + option.enable_thread_mgr = true; +#endif +#if WASM_ENABLE_TAIL_CALL != 0 + option.enable_tail_call = true; +#endif +#if WASM_ENABLE_SIMD != 0 + option.enable_simd = true; +#endif +#if WASM_ENABLE_REF_TYPES != 0 + option.enable_ref_types = true; +#endif + option.enable_aux_stack_check = true; +#if (WASM_ENABLE_PERF_PROFILING != 0) || (WASM_ENABLE_DUMP_CALL_STACK != 0) + option.enable_aux_stack_frame = true; +#endif + + module->comp_ctx = aot_create_comp_context(module->comp_data, &option); + if (!module->comp_ctx) { + aot_last_error = aot_get_last_error(); + bh_assert(aot_last_error != NULL); + set_error_buf(error_buf, error_buf_size, aot_last_error); + return false; + } + + if (!aot_compile_wasm(module->comp_ctx)) { + aot_last_error = aot_get_last_error(); + bh_assert(aot_last_error != NULL); + set_error_buf(error_buf, error_buf_size, aot_last_error); + return false; + } + +#if WASM_ENABLE_LAZY_JIT != 0 + for (i = 0; i < module->comp_data->func_count; i++) { + LLVMErrorRef error; + LLVMOrcJITTargetAddress func_addr = 0; + + snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i); + if ((error = LLVMOrcLLJITLookup(module->comp_ctx->orc_lazyjit, + &func_addr, func_name))) { + char *err_msg = LLVMGetErrorMessage(error); + set_error_buf_v(error_buf, error_buf_size, + "failed to compile orc jit function: %s", err_msg); + LLVMDisposeErrorMessage(err_msg); + return false; + } + /** + * No need to lock the func_ptr[func_idx] here as it is basic + * data type, the load/store for it can be finished by one cpu + * instruction, and there can be only one cpu instruction + * loading/storing at the same time. + */ + module->func_ptrs[i] = (void *)func_addr; + module->functions[i]->llvm_jit_func_ptr = (void *)func_addr; + } +#else + /* Resolve function addresses */ + bh_assert(module->comp_ctx->exec_engine); + for (i = 0; i < module->comp_data->func_count; i++) { + snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i); + if (!(module->func_ptrs[i] = (void *)LLVMGetFunctionAddress( + module->comp_ctx->exec_engine, func_name))) { + set_error_buf(error_buf, error_buf_size, + "failed to compile llvm mc jit function"); + return false; + } + module->functions[i]->llvm_jit_func_ptr = module->func_ptrs[i]; + } +#endif /* end of WASM_ENABLE_LAZY_JIT != 0 */ + + return true; +} +#endif /* end of WASM_ENABLE_JIT != 0 */ + #if WASM_ENABLE_REF_TYPES != 0 static bool get_table_elem_type(const WASMModule *module, uint32 table_idx, @@ -2185,6 +2324,8 @@ load_from_sections(WASMModule *module, WASMSection *sections, } } + calculate_global_data_offset(module); + #if WASM_ENABLE_FAST_JIT != 0 if (!(module->fast_jit_func_ptrs = loader_malloc(sizeof(void *) * module->function_count, error_buf, @@ -2197,6 +2338,12 @@ load_from_sections(WASMModule *module, WASMSection *sections, } #endif +#if WASM_ENABLE_JIT != 0 + if (!compile_llvm_jit_functions(module, error_buf, error_buf_size)) { + return false; + } +#endif /* end of WASM_ENABLE_JIT != 0 */ + #if WASM_ENABLE_MEMORY_TRACING != 0 wasm_runtime_dump_module_mem_consumption(module); #endif @@ -2526,6 +2673,15 @@ wasm_loader_unload(WASMModule *module) } #endif +#if WASM_ENABLE_JIT != 0 + if (module->func_ptrs) + wasm_runtime_free(module->func_ptrs); + if (module->comp_ctx) + aot_destroy_comp_context(module->comp_ctx); + if (module->comp_data) + aot_destroy_comp_data(module->comp_data); +#endif + wasm_runtime_free(module); } @@ -5035,8 +5191,7 @@ re_scan: bh_assert(type_index < module->type_count); block_type.is_value_type = false; block_type.u.type = module->types[type_index]; -#if WASM_ENABLE_FAST_INTERP == 0 && WASM_ENABLE_WAMR_COMPILER == 0 \ - && WASM_ENABLE_JIT == 0 +#if WASM_ENABLE_FAST_INTERP == 0 /* If block use type index as block type, change the opcode * to new extended opcode so that interpreter can resolve * the block quickly. diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index f085e785..7d00b5ea 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -19,6 +19,9 @@ #if WASM_ENABLE_DEBUG_INTERP != 0 #include "../libraries/debug-engine/debug_engine.h" #endif +#if WASM_ENABLE_JIT != 0 +#include "../aot/aot_runtime.h" +#endif static void set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) @@ -87,7 +90,7 @@ static WASMModuleInstance * get_sub_module_inst(const WASMModuleInstance *parent_module_inst, const WASMModule *sub_module) { - bh_list *sub_module_inst_list = parent_module_inst->sub_module_inst_list; + bh_list *sub_module_inst_list = parent_module_inst->e->sub_module_inst_list; WASMSubModInstNode *node = bh_list_first_elem(sub_module_inst_list); while (node && sub_module != node->module_inst->module) { @@ -126,7 +129,6 @@ memories_deinstantiate(WASMModuleInstance *module_inst, if (ref_count > 0) continue; } - os_mutex_destroy(&memories[i]->mem_lock); #endif if (memories[i]->heap_handle) { mem_allocator_destroy(memories[i]->heap_handle); @@ -146,7 +148,6 @@ memories_deinstantiate(WASMModuleInstance *module_inst, 8 * (uint64)BH_GB); #endif } - wasm_runtime_free(memories[i]); } } wasm_runtime_free(memories); @@ -155,13 +156,12 @@ memories_deinstantiate(WASMModuleInstance *module_inst, } static WASMMemoryInstance * -memory_instantiate(WASMModuleInstance *module_inst, uint32 num_bytes_per_page, - uint32 init_page_count, uint32 max_page_count, - uint32 heap_size, uint32 flags, char *error_buf, - uint32 error_buf_size) +memory_instantiate(WASMModuleInstance *module_inst, WASMMemoryInstance *memory, + uint32 num_bytes_per_page, uint32 init_page_count, + uint32 max_page_count, uint32 heap_size, uint32 flags, + char *error_buf, uint32 error_buf_size) { WASMModule *module = module_inst->module; - WASMMemoryInstance *memory; uint64 memory_data_size; uint32 heap_offset = num_bytes_per_page * init_page_count; uint32 inc_page_count, aux_heap_base, global_idx; @@ -250,10 +250,10 @@ memory_instantiate(WASMModuleInstance *module_inst, uint32 num_bytes_per_page, /* Adjust __heap_base global value */ global_idx = module->aux_heap_base_global_index; - bh_assert(module_inst->globals - && global_idx < module_inst->global_count); + bh_assert(module_inst->e->globals + && global_idx < module_inst->e->global_count); global_addr = module_inst->global_data - + module_inst->globals[global_idx].data_offset; + + module_inst->e->globals[global_idx].data_offset; *(uint32 *)global_addr = aux_heap_base; LOG_VERBOSE("Reset __heap_base global to %u", aux_heap_base); } @@ -296,12 +296,6 @@ memory_instantiate(WASMModuleInstance *module_inst, uint32 num_bytes_per_page, #endif bh_assert(memory_data_size <= 4 * (uint64)BH_GB); - /* Allocate memory space, addr data and global data */ - if (!(memory = runtime_malloc((uint64)sizeof(WASMMemoryInstance), error_buf, - error_buf_size))) { - return NULL; - } - #ifndef OS_ENABLE_HW_BOUND_CHECK if (memory_data_size > 0 && !(memory->memory_data = @@ -370,44 +364,40 @@ memory_instantiate(WASMModuleInstance *module_inst, uint32 num_bytes_per_page, } } -#if WASM_ENABLE_FAST_JIT != 0 +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 if (memory_data_size > 0) { #if UINTPTR_MAX == UINT64_MAX - memory->mem_bound_check_1byte = memory_data_size - 1; - memory->mem_bound_check_2bytes = memory_data_size - 2; - memory->mem_bound_check_4bytes = memory_data_size - 4; - memory->mem_bound_check_8bytes = memory_data_size - 8; - memory->mem_bound_check_16bytes = memory_data_size - 16; + memory->mem_bound_check_1byte.u64 = memory_data_size - 1; + memory->mem_bound_check_2bytes.u64 = memory_data_size - 2; + memory->mem_bound_check_4bytes.u64 = memory_data_size - 4; + memory->mem_bound_check_8bytes.u64 = memory_data_size - 8; + memory->mem_bound_check_16bytes.u64 = memory_data_size - 16; #else - memory->mem_bound_check_1byte = (uint32)memory_data_size - 1; - memory->mem_bound_check_2bytes = (uint32)memory_data_size - 2; - memory->mem_bound_check_4bytes = (uint32)memory_data_size - 4; - memory->mem_bound_check_8bytes = (uint32)memory_data_size - 8; - memory->mem_bound_check_16bytes = (uint32)memory_data_size - 16; + memory->mem_bound_check_1byte.u32[0] = (uint32)memory_data_size - 1; + memory->mem_bound_check_2bytes.u32[0] = (uint32)memory_data_size - 2; + memory->mem_bound_check_4bytes.u32[0] = (uint32)memory_data_size - 4; + memory->mem_bound_check_8bytes.u32[0] = (uint32)memory_data_size - 8; + memory->mem_bound_check_16bytes.u32[0] = (uint32)memory_data_size - 16; #endif } #endif #if WASM_ENABLE_SHARED_MEMORY != 0 - if (0 != os_mutex_init(&memory->mem_lock)) { - set_error_buf(error_buf, error_buf_size, "init mutex failed"); - goto fail4; - } if (is_shared_memory) { memory->is_shared = true; if (!shared_memory_set_memory_inst( (WASMModuleCommon *)module_inst->module, (WASMMemoryInstanceCommon *)memory)) { set_error_buf(error_buf, error_buf_size, "allocate memory failed"); - goto fail5; + goto fail4; } } #endif + LOG_VERBOSE("Memory instantiate success."); return memory; + #if WASM_ENABLE_SHARED_MEMORY != 0 -fail5: - os_mutex_destroy(&memory->mem_lock); fail4: if (heap_size > 0) mem_allocator_destroy(memory->heap_handle); @@ -426,8 +416,7 @@ fail2: os_munmap(mapped_mem, map_size); #endif fail1: - wasm_runtime_free(memory); - return NULL; + return memory; } /** @@ -449,9 +438,11 @@ memories_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, return NULL; } + memory = module_inst->global_table_data.memory_instances; + /* instantiate memories from import section */ import = module->import_memories; - for (i = 0; i < module->import_memory_count; i++, import++) { + for (i = 0; i < module->import_memory_count; i++, import++, memory++) { uint32 num_bytes_per_page = import->u.memory.num_bytes_per_page; uint32 init_page_count = import->u.memory.init_page_count; uint32 max_page_count = import->u.memory.max_page_count; @@ -469,7 +460,7 @@ memories_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, return NULL; } - if (!(memory = memories[mem_index++] = wasm_lookup_memory( + if (!(memories[mem_index++] = wasm_lookup_memory( module_inst_linked, import->u.memory.field_name))) { set_error_buf(error_buf, error_buf_size, "unknown memory"); memories_deinstantiate(module_inst, memories, memory_count); @@ -479,8 +470,8 @@ memories_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, else #endif { - if (!(memory = memories[mem_index++] = memory_instantiate( - module_inst, num_bytes_per_page, init_page_count, + if (!(memories[mem_index++] = memory_instantiate( + module_inst, memory, num_bytes_per_page, init_page_count, max_page_count, actual_heap_size, flags, error_buf, error_buf_size))) { memories_deinstantiate(module_inst, memories, memory_count); @@ -490,9 +481,9 @@ memories_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, } /* instantiate memories from memory section */ - for (i = 0; i < module->memory_count; i++) { - if (!(memory = memories[mem_index++] = memory_instantiate( - module_inst, module->memories[i].num_bytes_per_page, + for (i = 0; i < module->memory_count; i++, memory++) { + if (!(memories[mem_index++] = memory_instantiate( + module_inst, memory, module->memories[i].num_bytes_per_page, module->memories[i].init_page_count, module->memories[i].max_page_count, heap_size, module->memories[i].flags, error_buf, error_buf_size))) { @@ -510,15 +501,16 @@ memories_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, * Destroy table instances. */ static void -tables_deinstantiate(WASMTableInstance **tables, uint32 count) +tables_deinstantiate(WASMModuleInstance *module_inst) { - uint32 i; - if (tables) { - for (i = 0; i < count; i++) - if (tables[i]) - wasm_runtime_free(tables[i]); - wasm_runtime_free(tables); + if (module_inst->tables) { + wasm_runtime_free(module_inst->tables); } +#if WASM_ENABLE_MULTI_MODULE != 0 + if (module_inst->e->table_insts_linked) { + wasm_runtime_free(module_inst->e->table_insts_linked); + } +#endif } /** @@ -526,18 +518,32 @@ tables_deinstantiate(WASMTableInstance **tables, uint32 count) */ static WASMTableInstance ** tables_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, - char *error_buf, uint32 error_buf_size) + WASMTableInstance *first_table, char *error_buf, + uint32 error_buf_size) { WASMImport *import; - uint32 table_index = 0, i, - table_count = module->import_table_count + module->table_count; - uint64 total_size = sizeof(WASMTableInstance *) * (uint64)table_count; - WASMTableInstance **tables, *table; + uint32 table_index = 0, i; + uint32 table_count = module->import_table_count + module->table_count; + uint64 total_size = (uint64)sizeof(WASMTableInstance *) * table_count; + WASMTableInstance **tables, *table = first_table; +#if WASM_ENABLE_MULTI_MODULE != 0 + uint64 total_size_of_tables_linked = + (uint64)sizeof(WASMTableInstance *) * module->import_table_count; + WASMTableInstance **table_linked = NULL; +#endif if (!(tables = runtime_malloc(total_size, error_buf, error_buf_size))) { return NULL; } +#if WASM_ENABLE_MULTI_MODULE != 0 + if (module->import_table_count > 0 + && !(module_inst->e->table_insts_linked = table_linked = runtime_malloc( + total_size_of_tables_linked, error_buf, error_buf_size))) { + goto fail; + } +#endif + /* instantiate tables from import section */ import = module->import_tables; for (i = 0; i < module->import_table_count; i++, import++) { @@ -550,18 +556,16 @@ tables_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, if (!(module_inst_linked = get_sub_module_inst( module_inst, import->u.table.import_module))) { set_error_buf(error_buf, error_buf_size, "unknown table"); - tables_deinstantiate(tables, table_count); - return NULL; + goto fail; } if (!(table_inst_linked = wasm_lookup_table( module_inst_linked, import->u.table.field_name))) { set_error_buf(error_buf, error_buf_size, "unknown table"); - tables_deinstantiate(tables, table_count); - return NULL; + goto fail; } - total_size = offsetof(WASMTableInstance, base_addr); + total_size = offsetof(WASMTableInstance, elems); } else #endif @@ -572,40 +576,39 @@ tables_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, : import->u.table.init_size; /* it is a built-in table, every module has its own */ - total_size = offsetof(WASMTableInstance, base_addr); + total_size = offsetof(WASMTableInstance, elems); total_size += (uint64)max_size_fixed * sizeof(uint32); } - if (!(table = tables[table_index++] = - runtime_malloc(total_size, error_buf, error_buf_size))) { - tables_deinstantiate(tables, table_count); - return NULL; - } + tables[table_index++] = table; /* Set all elements to -1 to mark them as uninitialized elements */ memset(table, -1, (uint32)total_size); #if WASM_ENABLE_MULTI_MODULE != 0 - table->table_inst_linked = table_inst_linked; + *table_linked = table_inst_linked; if (table_inst_linked != NULL) { - table->elem_type = table_inst_linked->elem_type; table->cur_size = table_inst_linked->cur_size; table->max_size = table_inst_linked->max_size; } else #endif { - table->elem_type = import->u.table.elem_type; table->cur_size = import->u.table.init_size; table->max_size = max_size_fixed; } + + table = (WASMTableInstance *)((uint8 *)table + (uint32)total_size); +#if WASM_ENABLE_MULTI_MODULE != 0 + table_linked++; +#endif } /* instantiate tables from table section */ for (i = 0; i < module->table_count; i++) { uint32 max_size_fixed = 0; - total_size = offsetof(WASMTableInstance, base_addr); + total_size = offsetof(WASMTableInstance, elems); #if WASM_ENABLE_MULTI_MODULE != 0 /* in case, a module which imports this table will grow it */ max_size_fixed = module->tables[i].max_size; @@ -616,25 +619,24 @@ tables_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, #endif total_size += sizeof(uint32) * (uint64)max_size_fixed; - if (!(table = tables[table_index++] = - runtime_malloc(total_size, error_buf, error_buf_size))) { - tables_deinstantiate(tables, table_count); - return NULL; - } + tables[table_index++] = table; /* Set all elements to -1 to mark them as uninitialized elements */ memset(table, -1, (uint32)total_size); - table->elem_type = module->tables[i].elem_type; table->cur_size = module->tables[i].init_size; table->max_size = max_size_fixed; -#if WASM_ENABLE_MULTI_MODULE != 0 - table->table_inst_linked = NULL; -#endif + + table = (WASMTableInstance *)((uint8 *)table + (uint32)total_size); } bh_assert(table_index == table_count); (void)module_inst; return tables; +#if WASM_ENABLE_MULTI_MODULE != 0 +fail: + wasm_runtime_free(tables); + return NULL; +#endif } /** @@ -783,8 +785,7 @@ check_global_init_expr(const WASMModule *module, uint32 global_index, */ static WASMGlobalInstance * globals_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, - uint32 *p_global_data_size, char *error_buf, - uint32 error_buf_size) + char *error_buf, uint32 error_buf_size) { WASMImport *import; uint32 global_data_offset = 0; @@ -831,6 +832,9 @@ globals_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, &(global_import->global_data_linked), sizeof(WASMValue)); } +#if WASM_ENABLE_FAST_JIT != 0 + bh_assert(global_data_offset == global_import->data_offset); +#endif global->data_offset = global_data_offset; global_data_offset += wasm_value_type_size(global->type); @@ -843,8 +847,10 @@ globals_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, global->type = module->globals[i].type; global->is_mutable = module->globals[i].is_mutable; +#if WASM_ENABLE_FAST_JIT != 0 + bh_assert(global_data_offset == module->globals[i].data_offset); +#endif global->data_offset = global_data_offset; - global_data_offset += wasm_value_type_size(global->type); if (init_expr->init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) { @@ -871,7 +877,7 @@ globals_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, } bh_assert((uint32)(global - globals) == global_count); - *p_global_data_size = global_data_offset; + bh_assert(global_data_offset == module->global_data_size); (void)module_inst; return globals; fail: @@ -928,7 +934,7 @@ export_functions_instantiate(const WASMModule *module, for (i = 0; i < module->export_count; i++, export ++) if (export->kind == EXPORT_KIND_FUNC) { export_func->name = export->name; - export_func->function = &module_inst->functions[export->index]; + export_func->function = &module_inst->e->functions[export->index]; export_func++; } @@ -964,7 +970,7 @@ export_globals_instantiate(const WASMModule *module, for (i = 0; i < module->export_count; i++, export ++) if (export->kind == EXPORT_KIND_GLOBAL) { export_global->name = export->name; - export_global->global = &module_inst->globals[export->index]; + export_global->global = &module_inst->e->globals[export->index]; export_global++; } @@ -1034,7 +1040,7 @@ execute_memory_init_function(WASMModuleInstance *module_inst) static bool execute_start_function(WASMModuleInstance *module_inst) { - WASMFunctionInstance *func = module_inst->start_function; + WASMFunctionInstance *func = module_inst->e->start_function; if (!func) return true; @@ -1129,7 +1135,7 @@ sub_module_instantiate(WASMModule *module, WASMModuleInstance *module_inst, uint32 stack_size, uint32 heap_size, char *error_buf, uint32 error_buf_size) { - bh_list *sub_module_inst_list = module_inst->sub_module_inst_list; + bh_list *sub_module_inst_list = module_inst->e->sub_module_inst_list; WASMRegisteredModule *sub_module_list_node = bh_list_first_elem(module->import_module_list); @@ -1205,7 +1211,7 @@ sub_module_instantiate(WASMModule *module, WASMModuleInstance *module_inst, static void sub_module_deinstantiate(WASMModuleInstance *module_inst) { - bh_list *list = module_inst->sub_module_inst_list; + bh_list *list = module_inst->e->sub_module_inst_list; WASMSubModInstNode *node = bh_list_first_elem(list); while (node) { WASMSubModInstNode *next_node = bh_list_elem_next(node); @@ -1262,7 +1268,37 @@ check_linked_symbol(WASMModuleInstance *module_inst, char *error_buf, return true; } -#if WASM_ENABLE_FAST_JIT != 0 +#if WASM_ENABLE_JIT != 0 +static bool +init_func_ptrs(WASMModuleInstance *module_inst, WASMModule *module, + char *error_buf, uint32 error_buf_size) +{ + uint32 i; + void **func_ptrs; + uint64 total_size = (uint64)sizeof(void *) * module_inst->e->function_count; + + /* Allocate memory */ + if (!(func_ptrs = module_inst->func_ptrs = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + /* Set import function pointers */ + for (i = 0; i < module->import_function_count; i++, func_ptrs++) { + WASMFunctionImport *import_func = + &module->import_functions[i].u.function; + /* TODO: handle multi module */ + *func_ptrs = import_func->func_ptr_linked; + } + + /* Set defined function pointers */ + bh_memcpy_s(func_ptrs, sizeof(void *) * module->function_count, + module->func_ptrs, sizeof(void *) * module->function_count); + return true; +} +#endif /* end of WASM_ENABLE_JIT != 0 */ + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 static uint32 get_smallest_type_idx(WASMModule *module, WASMType *func_type) { @@ -1282,7 +1318,7 @@ init_func_type_indexes(WASMModuleInstance *module_inst, char *error_buf, uint32 error_buf_size) { uint32 i; - uint64 total_size = (uint64)sizeof(uint32) * module_inst->function_count; + uint64 total_size = (uint64)sizeof(uint32) * module_inst->e->function_count; /* Allocate memory */ if (!(module_inst->func_type_indexes = @@ -1290,8 +1326,8 @@ init_func_type_indexes(WASMModuleInstance *module_inst, char *error_buf, return false; } - for (i = 0; i < module_inst->function_count; i++) { - WASMFunctionInstance *func_inst = module_inst->functions + i; + for (i = 0; i < module_inst->e->function_count; i++) { + WASMFunctionInstance *func_inst = module_inst->e->functions + i; WASMType *func_type = func_inst->is_import_func ? func_inst->u.func_import->func_type : func_inst->u.func->func_type; @@ -1301,7 +1337,7 @@ init_func_type_indexes(WASMModuleInstance *module_inst, char *error_buf, return true; } -#endif +#endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 */ /** * Instantiate module @@ -1312,8 +1348,13 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, { WASMModuleInstance *module_inst; WASMGlobalInstance *globals = NULL, *global; - uint32 global_count, global_data_size = 0, i; - uint32 base_offset, length; + WASMTableInstance *first_table; + uint32 global_count, i; + uint32 base_offset, length, extra_info_offset; + uint32 module_inst_struct_size = + offsetof(WASMModuleInstance, global_table_data.bytes); + uint64 module_inst_mem_inst_size; + uint64 total_size, table_size = 0; uint8 *global_data, *global_data_end; #if WASM_ENABLE_MULTI_MODULE != 0 bool ret = false; @@ -1322,6 +1363,56 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, if (!module) return NULL; + /* Check the heap size */ + heap_size = align_uint(heap_size, 8); + if (heap_size > APP_HEAP_SIZE_MAX) + heap_size = APP_HEAP_SIZE_MAX; + + module_inst_mem_inst_size = + (uint64)sizeof(WASMMemoryInstance) + * (module->import_memory_count + module->memory_count); + +#if WASM_ENABLE_JIT != 0 + /* If the module dosen't have memory, reserve one mem_info space + with empty content to align with llvm jit compiler */ + if (module_inst_mem_inst_size == 0) + module_inst_mem_inst_size = (uint64)sizeof(WASMMemoryInstance); +#endif + + /* Size of module inst, memory instances and global data */ + total_size = (uint64)module_inst_struct_size + module_inst_mem_inst_size + + module->global_data_size; + + /* Calculate the size of table data */ + for (i = 0; i < module->import_table_count; i++) { + WASMTableImport *import_table = &module->import_tables[i].u.table; + table_size += offsetof(WASMTableInstance, elems); +#if WASM_ENABLE_MULTI_MODULE != 0 + table_size += (uint64)sizeof(uint32) * import_table->max_size; +#else + table_size += (uint64)sizeof(uint32) + * (import_table->possible_grow ? import_table->max_size + : import_table->init_size); +#endif + } + for (i = 0; i < module->table_count; i++) { + WASMTable *table = module->tables + i; + table_size += offsetof(WASMTableInstance, elems); +#if WASM_ENABLE_MULTI_MODULE != 0 + table_size += (uint64)sizeof(uint32) * table->max_size; +#else + table_size += + (uint64)sizeof(uint32) + * (table->possible_grow ? table->max_size : table->init_size); +#endif + } + total_size += table_size; + + /* The offset of WASMModuleInstanceExtra, make it 8-byte aligned */ + total_size = (total_size + 7LL) & ~7LL; + extra_info_offset = (uint32)total_size; + total_size += sizeof(WASMModuleInstanceExtra); + #if WASM_ENABLE_DEBUG_INTERP != 0 if (!is_sub_inst) { os_mutex_lock(&module->ref_count_lock); @@ -1335,14 +1426,10 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, } #endif - /* Check heap size */ - heap_size = align_uint(heap_size, 8); - if (heap_size > APP_HEAP_SIZE_MAX) - heap_size = APP_HEAP_SIZE_MAX; - - /* Allocate the memory */ - if (!(module_inst = runtime_malloc(sizeof(WASMModuleInstance), error_buf, - error_buf_size))) { + /* Allocate the memory for module instance with memory instances, + global data, table data appended at the end */ + if (!(module_inst = + runtime_malloc(total_size, error_buf, error_buf_size))) { #if WASM_ENABLE_DEBUG_INTERP != 0 if (!is_sub_inst) { os_mutex_lock(&module->ref_count_lock); @@ -1353,10 +1440,23 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, return NULL; } + module_inst->module_type = Wasm_Module_Bytecode; module_inst->module = module; + module_inst->e = + (WASMModuleInstanceExtra *)((uint8 *)module_inst + extra_info_offset); + +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (os_mutex_init(&module_inst->e->mem_lock) != 0) { + set_error_buf(error_buf, error_buf_size, + "create shared memory lock failed"); + goto fail; + } + module_inst->e->mem_lock_inited = true; +#endif #if WASM_ENABLE_MULTI_MODULE != 0 - module_inst->sub_module_inst_list = &module_inst->sub_module_inst_list_head; + module_inst->e->sub_module_inst_list = + &module_inst->e->sub_module_inst_list_head; ret = sub_module_instantiate(module, module_inst, stack_size, heap_size, error_buf, error_buf_size); if (!ret) { @@ -1375,59 +1475,62 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, /* Instantiate global firstly to get the mutable data size */ global_count = module->import_global_count + module->global_count; if (global_count - && !(globals = - globals_instantiate(module, module_inst, &global_data_size, - error_buf, error_buf_size))) { + && !(globals = globals_instantiate(module, module_inst, error_buf, + error_buf_size))) { goto fail; } - module_inst->global_count = global_count; - module_inst->globals = globals; + module_inst->e->global_count = global_count; + module_inst->e->globals = globals; + module_inst->global_data = (uint8 *)module_inst + module_inst_struct_size + + module_inst_mem_inst_size; + module_inst->global_data_size = module->global_data_size; + first_table = (WASMTableInstance *)(module_inst->global_data + + module->global_data_size); module_inst->memory_count = module->import_memory_count + module->memory_count; module_inst->table_count = module->import_table_count + module->table_count; - module_inst->function_count = + module_inst->e->function_count = module->import_function_count + module->function_count; /* export */ module_inst->export_func_count = get_export_count(module, EXPORT_KIND_FUNC); #if WASM_ENABLE_MULTI_MODULE != 0 - module_inst->export_tab_count = get_export_count(module, EXPORT_KIND_TABLE); - module_inst->export_mem_count = + module_inst->export_table_count = + get_export_count(module, EXPORT_KIND_TABLE); + module_inst->export_memory_count = get_export_count(module, EXPORT_KIND_MEMORY); - module_inst->export_glob_count = + module_inst->export_global_count = get_export_count(module, EXPORT_KIND_GLOBAL); #endif - if (global_count > 0) { - if (!(module_inst->global_data = runtime_malloc( - global_data_size, error_buf, error_buf_size))) { - goto fail; - } - } - /* Instantiate memories/tables/functions */ if ((module_inst->memory_count > 0 && !(module_inst->memories = memories_instantiate( module, module_inst, heap_size, error_buf, error_buf_size))) || (module_inst->table_count > 0 - && !(module_inst->tables = tables_instantiate( - module, module_inst, error_buf, error_buf_size))) - || (module_inst->function_count > 0 - && !(module_inst->functions = functions_instantiate( + && !(module_inst->tables = + tables_instantiate(module, module_inst, first_table, + error_buf, error_buf_size))) + || (module_inst->e->function_count > 0 + && !(module_inst->e->functions = functions_instantiate( module, module_inst, error_buf, error_buf_size))) || (module_inst->export_func_count > 0 && !(module_inst->export_functions = export_functions_instantiate( module, module_inst, module_inst->export_func_count, error_buf, error_buf_size))) #if WASM_ENABLE_MULTI_MODULE != 0 - || (module_inst->export_glob_count > 0 + || (module_inst->export_global_count > 0 && !(module_inst->export_globals = export_globals_instantiate( - module, module_inst, module_inst->export_glob_count, + module, module_inst, module_inst->export_global_count, error_buf, error_buf_size))) #endif -#if WASM_ENABLE_FAST_JIT != 0 - || (module_inst->function_count > 0 +#if WASM_ENABLE_JIT != 0 + || (module_inst->e->function_count > 0 + && !init_func_ptrs(module_inst, module, error_buf, error_buf_size)) +#endif +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 + || (module_inst->e->function_count > 0 && !init_func_type_indexes(module_inst, error_buf, error_buf_size)) #endif ) { @@ -1437,7 +1540,7 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, if (global_count > 0) { /* Initialize the global data */ global_data = module_inst->global_data; - global_data_end = global_data + global_data_size; + global_data_end = global_data + module->global_data_size; global = globals; for (i = 0; i < global_count; i++, global++) { switch (global->type) { @@ -1457,6 +1560,13 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, &global->initial_value.i64, sizeof(int64)); global_data += sizeof(int64); break; +#if WASM_ENABLE_SIMD != 0 + case VALUE_TYPE_V128: + bh_memcpy_s(global_data, (uint32)sizeof(V128), + &global->initial_value.v128, sizeof(V128)); + global_data += sizeof(V128); + break; +#endif default: bh_assert(0); } @@ -1469,9 +1579,6 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, } /* Initialize the memory data with data segment section */ - module_inst->default_memory = - module_inst->memory_count ? module_inst->memories[0] : NULL; - for (i = 0; i < module->data_seg_count; i++) { WASMMemoryInstance *memory = NULL; uint8 *memory_data = NULL; @@ -1554,30 +1661,41 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, } /* Initialize the table data with table segment section */ - module_inst->default_table = - module_inst->table_count ? module_inst->tables[0] : NULL; - /* in case there is no table */ for (i = 0; module_inst->table_count > 0 && i < module->table_seg_count; i++) { WASMTableSeg *table_seg = module->table_segments + i; /* has check it in loader */ WASMTableInstance *table = module_inst->tables[table_seg->table_index]; + uint32 *table_data; +#if WASM_ENABLE_REF_TYPES != 0 + uint8 tbl_elem_type; + uint32 tbl_init_size, tbl_max_size; +#endif + bh_assert(table); #if WASM_ENABLE_REF_TYPES != 0 - if (table->elem_type != VALUE_TYPE_FUNCREF - && table->elem_type != VALUE_TYPE_EXTERNREF) { + (void)wasm_runtime_get_table_inst_elem_type( + (WASMModuleInstanceCommon *)module_inst, table_seg->table_index, + &tbl_elem_type, &tbl_init_size, &tbl_max_size); + if (tbl_elem_type != VALUE_TYPE_FUNCREF + && tbl_elem_type != VALUE_TYPE_EXTERNREF) { set_error_buf(error_buf, error_buf_size, "elements segment does not fit"); goto fail; } + (void)tbl_init_size; + (void)tbl_max_size; #endif - uint32 *table_data = (uint32 *)table->base_addr; + table_data = table->elems; #if WASM_ENABLE_MULTI_MODULE != 0 - table_data = table->table_inst_linked - ? (uint32 *)table->table_inst_linked->base_addr - : table_data; + if (table_seg->table_index < module->import_table_count + && module_inst->e->table_insts_linked[table_seg->table_index]) { + table_data = + module_inst->e->table_insts_linked[table_seg->table_index] + ->elems; + } #endif bh_assert(table_data); @@ -1586,7 +1704,6 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, continue; #endif - /* init vec(funcidx) or vec(expr) */ #if WASM_ENABLE_REF_TYPES != 0 bh_assert(table_seg->base_offset.init_expr_type == INIT_EXPR_TYPE_I32_CONST @@ -1603,6 +1720,7 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, == INIT_EXPR_TYPE_GET_GLOBAL); #endif + /* init vec(funcidx) or vec(expr) */ if (table_seg->base_offset.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) { if (!check_global_init_expr(module, @@ -1665,9 +1783,6 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, table_seg->func_indexes, (uint32)(length * sizeof(uint32))); } - /* module instance type */ - module_inst->module_type = Wasm_Module_Bytecode; - /* Initialize the thread related data */ if (stack_size == 0) stack_size = DEFAULT_WASM_STACK_SIZE; @@ -1678,18 +1793,18 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, module_inst->default_wasm_stack_size = stack_size; if (module->malloc_function != (uint32)-1) { - module_inst->malloc_function = - &module_inst->functions[module->malloc_function]; + module_inst->e->malloc_function = + &module_inst->e->functions[module->malloc_function]; } if (module->free_function != (uint32)-1) { - module_inst->free_function = - &module_inst->functions[module->free_function]; + module_inst->e->free_function = + &module_inst->e->functions[module->free_function]; } if (module->retain_function != (uint32)-1) { - module_inst->retain_function = - &module_inst->functions[module->retain_function]; + module_inst->e->retain_function = + &module_inst->e->functions[module->retain_function]; } #if WASM_ENABLE_LIBC_WASI != 0 @@ -1714,8 +1829,8 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, if (module->start_function != (uint32)-1) { /* TODO: fix start function can be import function issue */ if (module->start_function >= module->import_function_count) - module_inst->start_function = - &module_inst->functions[module->start_function]; + module_inst->e->start_function = + &module_inst->e->functions[module->start_function]; } /* Execute __post_instantiate function */ @@ -1763,7 +1878,12 @@ wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst) if (!module_inst) return; -#if WASM_ENABLE_FAST_JIT != 0 +#if WASM_ENABLE_JIT != 0 + if (module_inst->func_ptrs) + wasm_runtime_free(module_inst->func_ptrs); +#endif + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 if (module_inst->func_type_indexes) wasm_runtime_free(module_inst->func_type_indexes); #endif @@ -1790,18 +1910,15 @@ wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst) wasm_runtime_free(module_inst->import_func_ptrs); } - tables_deinstantiate(module_inst->tables, module_inst->table_count); - functions_deinstantiate(module_inst->functions, - module_inst->function_count); - globals_deinstantiate(module_inst->globals); + tables_deinstantiate(module_inst); + functions_deinstantiate(module_inst->e->functions, + module_inst->e->function_count); + globals_deinstantiate(module_inst->e->globals); export_functions_deinstantiate(module_inst->export_functions); #if WASM_ENABLE_MULTI_MODULE != 0 export_globals_deinstantiate(module_inst->export_globals); #endif - if (module_inst->global_data) - wasm_runtime_free(module_inst->global_data); - #if WASM_ENABLE_REF_TYPES != 0 wasm_externref_cleanup((WASMModuleInstanceCommon *)module_inst); #endif @@ -1825,6 +1942,11 @@ wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst) } #endif +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (module_inst->e->mem_lock_inited) + os_mutex_destroy(&module_inst->e->mem_lock); +#endif + wasm_runtime_free(module_inst); } @@ -1845,7 +1967,7 @@ WASMGlobalInstance * wasm_lookup_global(const WASMModuleInstance *module_inst, const char *name) { uint32 i; - for (i = 0; i < module_inst->export_glob_count; i++) + for (i = 0; i < module_inst->export_global_count; i++) if (!strcmp(module_inst->export_globals[i].name, name)) return module_inst->export_globals[i].global; return NULL; @@ -1894,113 +2016,6 @@ clear_wasi_proc_exit_exception(WASMModuleInstance *module_inst) #ifdef OS_ENABLE_HW_BOUND_CHECK -#ifndef BH_PLATFORM_WINDOWS -void -wasm_signal_handler(WASMSignalInfo *sig_info) -{ - WASMExecEnv *exec_env_tls = sig_info->exec_env_tls; - void *sig_addr = sig_info->sig_addr; - WASMModuleInstance *module_inst; - WASMMemoryInstance *memory_inst; - WASMJmpBuf *jmpbuf_node; - uint8 *mapped_mem_start_addr = NULL; - uint8 *mapped_mem_end_addr = NULL; - uint8 *stack_min_addr; - uint32 page_size; - uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; - - /* Check whether current thread is running wasm function */ - if (exec_env_tls && exec_env_tls->handle == os_self_thread() - && (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) { - /* Get mapped mem info of current instance */ - module_inst = (WASMModuleInstance *)exec_env_tls->module_inst; - /* Get the default memory instance */ - memory_inst = module_inst->default_memory; - if (memory_inst) { - mapped_mem_start_addr = (uint8 *)memory_inst->memory_data; - mapped_mem_end_addr = - (uint8 *)memory_inst->memory_data + 8 * (uint64)BH_GB; - } - - /* Get stack info of current thread */ - page_size = os_getpagesize(); - stack_min_addr = os_thread_get_stack_boundary(); - - if (memory_inst - && (mapped_mem_start_addr <= (uint8 *)sig_addr - && (uint8 *)sig_addr < mapped_mem_end_addr)) { - /* The address which causes segmentation fault is inside - the memory instance's guard regions */ - wasm_set_exception(module_inst, "out of bounds memory access"); - os_longjmp(jmpbuf_node->jmpbuf, 1); - } - else if (stack_min_addr - page_size <= (uint8 *)sig_addr - && (uint8 *)sig_addr - < stack_min_addr + page_size * guard_page_count) { - /* The address which causes segmentation fault is inside - native thread's guard page */ - wasm_set_exception(module_inst, "native stack overflow"); - os_longjmp(jmpbuf_node->jmpbuf, 1); - } - } -} -#else /* else of BH_PLATFORM_WINDOWS */ -LONG -wasm_exception_handler(WASMSignalInfo *sig_info) -{ - WASMExecEnv *exec_env_tls = sig_info->exec_env_tls; - EXCEPTION_POINTERS *exce_info = sig_info->exce_info; - PEXCEPTION_RECORD ExceptionRecord = exce_info->ExceptionRecord; - uint8 *sig_addr = (uint8 *)ExceptionRecord->ExceptionInformation[1]; - WASMModuleInstance *module_inst; - WASMMemoryInstance *memory_inst; - WASMJmpBuf *jmpbuf_node; - uint8 *mapped_mem_start_addr = NULL; - uint8 *mapped_mem_end_addr = NULL; - uint32 page_size = os_getpagesize(); - - if (exec_env_tls && exec_env_tls->handle == os_self_thread() - && (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) { - module_inst = (WASMModuleInstance *)exec_env_tls->module_inst; - if (ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { - /* Get the default memory instance */ - memory_inst = module_inst->default_memory; - if (memory_inst) { - mapped_mem_start_addr = (uint8 *)memory_inst->memory_data; - mapped_mem_end_addr = - (uint8 *)memory_inst->memory_data + 8 * (uint64)BH_GB; - if (mapped_mem_start_addr <= (uint8 *)sig_addr - && (uint8 *)sig_addr < mapped_mem_end_addr) { - /* The address which causes segmentation fault is inside - the memory instance's guard regions. - Set exception and let the wasm func continue to run, when - the wasm func returns, the caller will check whether the - exception is thrown and return to runtime. */ - wasm_set_exception(module_inst, - "out of bounds memory access"); - /* Skip current instruction */ - return EXCEPTION_CONTINUE_SEARCH; - } - } - } - else if (ExceptionRecord->ExceptionCode == EXCEPTION_STACK_OVERFLOW) { - /* Set stack overflow exception and let the wasm func continue - to run, when the wasm func returns, the caller will check - whether the exception is thrown and return to runtime, and - the damaged stack will be recovered by _resetstkoflw(). */ - wasm_set_exception(module_inst, "native stack overflow"); - return EXCEPTION_CONTINUE_SEARCH; - } - } - - os_printf("Unhandled exception thrown: exception code: 0x%lx, " - "exception address: %p, exception information: %p\n", - ExceptionRecord->ExceptionCode, ExceptionRecord->ExceptionAddress, - sig_addr); - return EXCEPTION_CONTINUE_SEARCH; -} -#endif /* end of BH_PLATFORM_WINDOWS */ - static void call_wasm_with_hw_bound_check(WASMModuleInstance *module_inst, WASMExecEnv *exec_env, @@ -2138,42 +2153,6 @@ wasm_create_exec_env_and_call_function(WASMModuleInstance *module_inst, return ret; } -bool -wasm_create_exec_env_singleton(WASMModuleInstance *module_inst) -{ - WASMExecEnv *exec_env = NULL; - - if (module_inst->exec_env_singleton) { - return true; - } - - exec_env = wasm_exec_env_create((WASMModuleInstanceCommon *)module_inst, - module_inst->default_wasm_stack_size); - if (exec_env) - module_inst->exec_env_singleton = exec_env; - - return exec_env ? true : false; -} - -void -wasm_set_exception(WASMModuleInstance *module_inst, const char *exception) -{ - if (exception) - snprintf(module_inst->cur_exception, sizeof(module_inst->cur_exception), - "Exception: %s", exception); - else - module_inst->cur_exception[0] = '\0'; -} - -const char * -wasm_get_exception(WASMModuleInstance *module_inst) -{ - if (module_inst->cur_exception[0] == '\0') - return NULL; - else - return module_inst->cur_exception; -} - #if WASM_ENABLE_PERF_PROFILING != 0 void wasm_dump_perf_profiling(const WASMModuleInstance *module_inst) @@ -2184,8 +2163,8 @@ wasm_dump_perf_profiling(const WASMModuleInstance *module_inst) uint32 i, j; os_printf("Performance profiler data:\n"); - for (i = 0; i < module_inst->function_count; i++) { - func_inst = module_inst->functions + i; + for (i = 0; i < module_inst->e->function_count; i++) { + func_inst = module_inst->e->functions + i; if (func_inst->is_import_func) { func_name = func_inst->u.func_import->field_name; } @@ -2209,13 +2188,13 @@ wasm_dump_perf_profiling(const WASMModuleInstance *module_inst) os_printf(" func %s, execution time: %.3f ms, execution count: %d " "times\n", func_name, - module_inst->functions[i].total_exec_time / 1000.0f, - module_inst->functions[i].total_exec_cnt); + module_inst->e->functions[i].total_exec_time / 1000.0f, + module_inst->e->functions[i].total_exec_cnt); else os_printf(" func %d, execution time: %.3f ms, execution count: %d " "times\n", - i, module_inst->functions[i].total_exec_time / 1000.0f, - module_inst->functions[i].total_exec_cnt); + i, module_inst->e->functions[i].total_exec_time / 1000.0f, + module_inst->e->functions[i].total_exec_cnt); } } #endif @@ -2224,7 +2203,7 @@ uint32 wasm_module_malloc(WASMModuleInstance *module_inst, uint32 size, void **p_native_addr) { - WASMMemoryInstance *memory = module_inst->default_memory; + WASMMemoryInstance *memory = wasm_get_default_memory(module_inst); uint8 *addr = NULL; uint32 offset = 0; @@ -2236,15 +2215,15 @@ wasm_module_malloc(WASMModuleInstance *module_inst, uint32 size, if (memory->heap_handle) { addr = mem_allocator_malloc(memory->heap_handle, size); } - else if (module_inst->malloc_function && module_inst->free_function) { - if (!execute_malloc_function(module_inst, module_inst->malloc_function, - module_inst->retain_function, size, - &offset)) { + else if (module_inst->e->malloc_function && module_inst->e->free_function) { + if (!execute_malloc_function( + module_inst, module_inst->e->malloc_function, + module_inst->e->retain_function, size, &offset)) { return 0; } /* If we use app's malloc function, the default memory may be changed while memory growing */ - memory = module_inst->default_memory; + memory = wasm_get_default_memory(module_inst); addr = offset ? memory->memory_data + offset : NULL; } @@ -2269,7 +2248,7 @@ uint32 wasm_module_realloc(WASMModuleInstance *module_inst, uint32 ptr, uint32 size, void **p_native_addr) { - WASMMemoryInstance *memory = module_inst->default_memory; + WASMMemoryInstance *memory = wasm_get_default_memory(module_inst); uint8 *addr = NULL; if (!memory) { @@ -2304,7 +2283,7 @@ void wasm_module_free(WASMModuleInstance *module_inst, uint32 ptr) { if (ptr) { - WASMMemoryInstance *memory = module_inst->default_memory; + WASMMemoryInstance *memory = wasm_get_default_memory(module_inst); uint8 *addr; if (!memory) { @@ -2317,10 +2296,11 @@ wasm_module_free(WASMModuleInstance *module_inst, uint32 ptr) && addr < memory->heap_data_end) { mem_allocator_free(memory->heap_handle, addr); } - else if (module_inst->malloc_function && module_inst->free_function - && memory->memory_data <= addr + else if (module_inst->e->malloc_function + && module_inst->e->free_function && memory->memory_data <= addr && addr < memory->memory_data_end) { - execute_free_function(module_inst, module_inst->free_function, ptr); + execute_free_function(module_inst, module_inst->e->free_function, + ptr); } } } @@ -2333,338 +2313,22 @@ wasm_module_dup_data(WASMModuleInstance *module_inst, const char *src, uint32 buffer_offset = wasm_module_malloc(module_inst, size, (void **)&buffer); if (buffer_offset != 0) { - buffer = wasm_addr_app_to_native(module_inst, buffer_offset); + buffer = wasm_runtime_addr_app_to_native( + (WASMModuleInstanceCommon *)module_inst, buffer_offset); bh_memcpy_s(buffer, size, src, size); } return buffer_offset; } -bool -wasm_validate_app_addr(WASMModuleInstance *module_inst, uint32 app_offset, - uint32 size) -{ - WASMMemoryInstance *memory = module_inst->default_memory; - uint32 memory_data_size; - - if (!memory) { - goto fail; - } - - memory_data_size = memory->num_bytes_per_page * memory->cur_page_count; - - /* integer overflow check */ - if (app_offset > UINT32_MAX - size) { - goto fail; - } - - if (app_offset + size <= memory_data_size) { - return true; - } -fail: - wasm_set_exception(module_inst, "out of bounds memory access"); - return false; -} - -bool -wasm_validate_native_addr(WASMModuleInstance *module_inst, void *native_ptr, - uint32 size) -{ - WASMMemoryInstance *memory = module_inst->default_memory; - uint8 *addr = (uint8 *)native_ptr; - - if (!memory) { - goto fail; - } - - /* integer overflow check */ - if ((uintptr_t)addr > UINTPTR_MAX - size) { - goto fail; - } - - if (memory->memory_data <= addr && addr + size <= memory->memory_data_end) { - return true; - } -fail: - wasm_set_exception(module_inst, "out of bounds memory access"); - return false; -} - -void * -wasm_addr_app_to_native(WASMModuleInstance *module_inst, uint32 app_offset) -{ - WASMMemoryInstance *memory = module_inst->default_memory; - uint8 *addr; - - if (!memory) - return NULL; - - addr = memory->memory_data + app_offset; - - if (memory->memory_data <= addr && addr < memory->memory_data_end) - return addr; - return NULL; -} - -uint32 -wasm_addr_native_to_app(WASMModuleInstance *module_inst, void *native_ptr) -{ - WASMMemoryInstance *memory = module_inst->default_memory; - uint8 *addr = (uint8 *)native_ptr; - - if (!memory) - return 0; - - if (memory->memory_data <= addr && addr < memory->memory_data_end) - return (uint32)(addr - memory->memory_data); - return 0; -} - -bool -wasm_get_app_addr_range(WASMModuleInstance *module_inst, uint32 app_offset, - uint32 *p_app_start_offset, uint32 *p_app_end_offset) -{ - WASMMemoryInstance *memory = module_inst->default_memory; - uint32 memory_data_size; - - if (!memory) - return false; - - memory_data_size = memory->num_bytes_per_page * memory->cur_page_count; - - if (app_offset < memory_data_size) { - if (p_app_start_offset) - *p_app_start_offset = 0; - if (p_app_end_offset) - *p_app_end_offset = memory_data_size; - return true; - } - return false; -} - -bool -wasm_get_native_addr_range(WASMModuleInstance *module_inst, uint8 *native_ptr, - uint8 **p_native_start_addr, - uint8 **p_native_end_addr) -{ - WASMMemoryInstance *memory = module_inst->default_memory; - uint8 *addr = (uint8 *)native_ptr; - - if (!memory) - return false; - - if (memory->memory_data <= addr && addr < memory->memory_data_end) { - if (p_native_start_addr) - *p_native_start_addr = memory->memory_data; - if (p_native_end_addr) - *p_native_end_addr = memory->memory_data_end; - return true; - } - return false; -} - -#ifndef OS_ENABLE_HW_BOUND_CHECK -bool -wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) -{ - WASMMemoryInstance *memory = module->default_memory; - uint8 *memory_data_old, *memory_data_new, *heap_data_old; - uint32 num_bytes_per_page, heap_size, total_size_old; - uint32 cur_page_count, max_page_count, total_page_count; - uint64 total_size_new; - bool ret = true; - - if (!memory) - return false; - - heap_data_old = memory->heap_data; - heap_size = (uint32)(memory->heap_data_end - memory->heap_data); - - memory_data_old = memory->memory_data; - total_size_old = memory->memory_data_size; - - num_bytes_per_page = memory->num_bytes_per_page; - cur_page_count = memory->cur_page_count; - max_page_count = memory->max_page_count; - total_page_count = inc_page_count + cur_page_count; - total_size_new = num_bytes_per_page * (uint64)total_page_count; - - if (inc_page_count <= 0) - /* No need to enlarge memory */ - return true; - - if (total_page_count < cur_page_count /* integer overflow */ - || total_page_count > max_page_count) { - return false; - } - - bh_assert(total_size_new <= 4 * (uint64)BH_GB); - if (total_size_new > UINT32_MAX) { - /* Resize to 1 page with size 4G-1 */ - num_bytes_per_page = UINT32_MAX; - total_page_count = max_page_count = 1; - total_size_new = UINT32_MAX; - } - -#if WASM_ENABLE_SHARED_MEMORY != 0 - if (memory->is_shared) { - memory->num_bytes_per_page = num_bytes_per_page; - memory->cur_page_count = total_page_count; - memory->max_page_count = max_page_count; - /* No need to update memory->memory_data_size as it is - initialized with the maximum memory data size for - shared memory */ - return true; - } -#endif - - if (heap_size > 0) { - if (mem_allocator_is_heap_corrupted(memory->heap_handle)) { - wasm_runtime_show_app_heap_corrupted_prompt(); - return false; - } - } - - if (!(memory_data_new = - wasm_runtime_realloc(memory_data_old, (uint32)total_size_new))) { - if (!(memory_data_new = wasm_runtime_malloc((uint32)total_size_new))) { - return false; - } - if (memory_data_old) { - bh_memcpy_s(memory_data_new, (uint32)total_size_new, - memory_data_old, total_size_old); - wasm_runtime_free(memory_data_old); - } - } - - memset(memory_data_new + total_size_old, 0, - (uint32)total_size_new - total_size_old); - - if (heap_size > 0) { - if (mem_allocator_migrate(memory->heap_handle, - (char *)heap_data_old - + (memory_data_new - memory_data_old), - heap_size) - != 0) { - /* Don't return here as memory->memory_data is obsolete and - must be updated to be correctly used later. */ - ret = false; - } - } - - memory->heap_data = memory_data_new + (heap_data_old - memory_data_old); - memory->heap_data_end = memory->heap_data + heap_size; - - memory->num_bytes_per_page = num_bytes_per_page; - memory->cur_page_count = total_page_count; - memory->max_page_count = max_page_count; - memory->memory_data_size = (uint32)total_size_new; - - memory->memory_data = memory_data_new; - memory->memory_data_end = memory_data_new + (uint32)total_size_new; - -#if WASM_ENABLE_FAST_JIT != 0 -#if UINTPTR_MAX == UINT64_MAX - memory->mem_bound_check_1byte = total_size_new - 1; - memory->mem_bound_check_2bytes = total_size_new - 2; - memory->mem_bound_check_4bytes = total_size_new - 4; - memory->mem_bound_check_8bytes = total_size_new - 8; - memory->mem_bound_check_16bytes = total_size_new - 16; -#else - memory->mem_bound_check_1byte = (uint32)total_size_new - 1; - memory->mem_bound_check_2bytes = (uint32)total_size_new - 2; - memory->mem_bound_check_4bytes = (uint32)total_size_new - 4; - memory->mem_bound_check_8bytes = (uint32)total_size_new - 8; - memory->mem_bound_check_16bytes = (uint32)total_size_new - 16; -#endif -#endif - - return ret; -} -#else -bool -wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) -{ - WASMMemoryInstance *memory = module->default_memory; - uint32 num_bytes_per_page, total_size_old; - uint32 cur_page_count, max_page_count, total_page_count; - uint64 total_size_new; - - if (!memory) - return false; - - num_bytes_per_page = memory->num_bytes_per_page; - cur_page_count = memory->cur_page_count; - max_page_count = memory->max_page_count; - total_size_old = num_bytes_per_page * cur_page_count; - total_page_count = inc_page_count + cur_page_count; - total_size_new = num_bytes_per_page * (uint64)total_page_count; - - if (inc_page_count <= 0) - /* No need to enlarge memory */ - return true; - - if (total_page_count < cur_page_count /* integer overflow */ - || total_page_count > max_page_count) { - return false; - } - - bh_assert(total_size_new <= 4 * (uint64)BH_GB); - if (total_size_new > UINT32_MAX) { - /* Resize to 1 page with size 4G-1 */ - num_bytes_per_page = UINT32_MAX; - total_page_count = max_page_count = 1; - total_size_new = UINT32_MAX; - } - -#ifdef BH_PLATFORM_WINDOWS - if (!os_mem_commit(memory->memory_data_end, - (uint32)total_size_new - total_size_old, - MMAP_PROT_READ | MMAP_PROT_WRITE)) { - return false; - } -#endif - - if (os_mprotect(memory->memory_data_end, - (uint32)total_size_new - total_size_old, - MMAP_PROT_READ | MMAP_PROT_WRITE) - != 0) { -#ifdef BH_PLATFORM_WINDOWS - os_mem_decommit(memory->memory_data_end, - (uint32)total_size_new - total_size_old); -#endif - return false; - } - - /* The increased pages are filled with zero by the OS when os_mmap, - no need to memset it again here */ - - memory->num_bytes_per_page = num_bytes_per_page; - memory->cur_page_count = total_page_count; - memory->max_page_count = max_page_count; - memory->memory_data_size = (uint32)total_size_new; - memory->memory_data_end = memory->memory_data + (uint32)total_size_new; - -#if WASM_ENABLE_FAST_JIT != 0 - memory->mem_bound_check_1byte = total_size_new - 1; - memory->mem_bound_check_2bytes = total_size_new - 2; - memory->mem_bound_check_4bytes = total_size_new - 4; - memory->mem_bound_check_8bytes = total_size_new - 8; - memory->mem_bound_check_16bytes = total_size_new - 16; -#endif - - return true; -} -#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ - #if WASM_ENABLE_REF_TYPES != 0 bool wasm_enlarge_table(WASMModuleInstance *module_inst, uint32 table_idx, - uint32 inc_entries, uint32 init_val) + uint32 inc_size, uint32 init_val) { - uint32 entry_count, *new_table_data_start, i; + uint32 total_size, *new_table_data_start, i; WASMTableInstance *table_inst; - if (!inc_entries) { + if (!inc_size) { return true; } @@ -2674,24 +2338,22 @@ wasm_enlarge_table(WASMModuleInstance *module_inst, uint32 table_idx, return false; } - if (inc_entries > UINT32_MAX - table_inst->cur_size) { + if (inc_size > UINT32_MAX - table_inst->cur_size) { return false; } - entry_count = table_inst->cur_size + inc_entries; - if (entry_count > table_inst->max_size) { + total_size = table_inst->cur_size + inc_size; + if (total_size > table_inst->max_size) { return false; } /* fill in */ - new_table_data_start = - (uint32 *)((uint8 *)table_inst + offsetof(WASMTableInstance, base_addr)) - + table_inst->cur_size; - for (i = 0; i < inc_entries; ++i) { + new_table_data_start = table_inst->elems + table_inst->cur_size; + for (i = 0; i < inc_size; ++i) { new_table_data_start[i] = init_val; } - table_inst->cur_size = entry_count; + table_inst->cur_size = total_size; return true; } #endif /* WASM_ENABLE_REF_TYPES != 0 */ @@ -2719,11 +2381,7 @@ call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, goto got_exception; } - /** - * please be aware that table_inst->base_addr may point - * to another module's table - **/ - func_idx = ((uint32 *)table_inst->base_addr)[elem_idx]; + func_idx = table_inst->elems[elem_idx]; if (func_idx == NULL_REF) { wasm_set_exception(module_inst, "uninitialized element"); goto got_exception; @@ -2732,12 +2390,12 @@ call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, /** * we insist to call functions owned by the module itself **/ - if (func_idx >= module_inst->function_count) { + if (func_idx >= module_inst->e->function_count) { wasm_set_exception(module_inst, "unknown function"); goto got_exception; } - func_inst = module_inst->functions + func_idx; + func_inst = module_inst->e->functions + func_idx; if (check_type_idx) { WASMType *cur_type = module_inst->module->types[type_idx]; @@ -2770,16 +2428,6 @@ wasm_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, return call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, false, 0); } -#if WASM_ENABLE_FAST_JIT != 0 -bool -jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, - uint32 type_idx, uint32 argc, uint32 argv[]) -{ - return call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, true, - type_idx); -} -#endif - #if WASM_ENABLE_THREAD_MGR != 0 bool wasm_set_aux_stack(WASMExecEnv *exec_env, uint32 start_offset, uint32 size) @@ -2799,8 +2447,9 @@ wasm_set_aux_stack(WASMExecEnv *exec_env, uint32 start_offset, uint32 size) if (stack_top_idx != (uint32)-1) { /* The aux stack top is a wasm global, set the initial value for the global */ - uint8 *global_addr = module_inst->global_data - + module_inst->globals[stack_top_idx].data_offset; + uint8 *global_addr = + module_inst->global_data + + module_inst->e->globals[stack_top_idx].data_offset; *(int32 *)global_addr = start_offset; /* The aux stack boundary is a constant value, set the value to exec_env */ @@ -2917,48 +2566,34 @@ wasm_get_module_inst_mem_consumption(const WASMModuleInstance *module_inst, memset(mem_conspn, 0, sizeof(*mem_conspn)); - mem_conspn->module_inst_struct_size = sizeof(WASMModuleInstance); + mem_conspn->module_inst_struct_size = (uint8 *)module_inst->e + - (uint8 *)module_inst + + sizeof(WASMModuleInstanceExtra); mem_conspn->memories_size = sizeof(WASMMemoryInstance *) * module_inst->memory_count; for (i = 0; i < module_inst->memory_count; i++) { WASMMemoryInstance *memory = module_inst->memories[i]; - size = sizeof(WASMMemoryInstance) - + memory->num_bytes_per_page * memory->cur_page_count; + size = memory->num_bytes_per_page * memory->cur_page_count; mem_conspn->memories_size += size; mem_conspn->app_heap_size += memory->heap_data_end - memory->heap_data; /* size of app heap structure */ mem_conspn->memories_size += mem_allocator_get_heap_struct_size(); + /* Module instance structures have been appened into the end of + module instance */ } mem_conspn->tables_size = sizeof(WASMTableInstance *) * module_inst->table_count; - for (i = 0; i < module_inst->table_count; i++) { - WASMTableInstance *table = module_inst->tables[i]; -#if WASM_ENABLE_MULTI_MODULE != 0 - if (table->table_inst_linked) { - size = offsetof(WASMTableInstance, base_addr); - } - else -#endif - { - size = offsetof(WASMTableInstance, base_addr) - + sizeof(uint32) * table->cur_size; - } - mem_conspn->tables_size += size; - } + /* Table instance structures and table elements have been appened into + the end of module instance */ mem_conspn->functions_size = - sizeof(WASMFunctionInstance) * module_inst->function_count; + sizeof(WASMFunctionInstance) * module_inst->e->function_count; mem_conspn->globals_size = - sizeof(WASMGlobalInstance) * module_inst->global_count; - if (module_inst->global_count > 0) { - WASMGlobalInstance *global = - &module_inst->globals[module_inst->global_count - 1]; - mem_conspn->globals_size += - global->data_offset + wasm_value_type_size(global->type); - } + sizeof(WASMGlobalInstance) * module_inst->e->global_count; + /* Global data has been appened into the end of module instance */ mem_conspn->exports_size = sizeof(WASMExportFuncInstance) * module_inst->export_func_count; @@ -3016,7 +2651,7 @@ wasm_interp_create_call_stack(struct WASMExecEnv *exec_env) /* place holder, will overwrite it in wasm_c_api */ frame.instance = module_inst; frame.module_offset = 0; - frame.func_index = (uint32)(func_inst - module_inst->functions); + frame.func_index = (uint32)(func_inst - module_inst->e->functions); func_code_base = wasm_get_func_code(func_inst); if (!cur_frame->ip || !func_code_base) { @@ -3132,3 +2767,396 @@ wasm_interp_dump_call_stack(struct WASMExecEnv *exec_env, bool print, char *buf, return total_len + 1; } #endif /* end of WASM_ENABLE_DUMP_CALL_STACK */ + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 +void +jit_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id) +{ + wasm_set_exception_with_id(module_inst, id); +} + +bool +jit_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, + uint32 app_buf_addr, uint32 app_buf_size, + void **p_native_addr) +{ + return wasm_check_app_addr_and_convert(module_inst, is_str, app_buf_addr, + app_buf_size, p_native_addr); +} +#endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 */ + +#if WASM_ENABLE_FAST_JIT != 0 +bool +fast_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 type_idx, uint32 argc, uint32 *argv) +{ + return call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, true, + type_idx); +} +#endif + +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + +bool +llvm_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 argc, uint32 *argv) +{ +#if WASM_ENABLE_JIT != 0 + if (Wasm_Module_AoT == exec_env->module_inst->module_type) { + return aot_call_indirect(exec_env, tbl_idx, elem_idx, argc, argv); + } +#endif + return call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, false, 0); +} + +bool +llvm_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc, + uint32 *argv) +{ + WASMModuleInstance *module_inst; + WASMModule *module; + uint32 *func_type_indexes; + uint32 func_type_idx; + WASMType *func_type; + void *func_ptr; + WASMFunctionImport *import_func; + const char *signature; + void *attachment; + char buf[96]; + +#if WASM_ENABLE_JIT != 0 + if (Wasm_Module_AoT == exec_env->module_inst->module_type) { + return aot_invoke_native(exec_env, func_idx, argc, argv); + } +#endif + + module_inst = (WASMModuleInstance *)wasm_runtime_get_module_inst(exec_env); + module = module_inst->module; + func_type_indexes = module_inst->func_type_indexes; + func_type_idx = func_type_indexes[func_idx]; + func_type = module->types[func_type_idx]; + func_ptr = module_inst->func_ptrs[func_idx]; + + bh_assert(func_idx < module->import_function_count); + + import_func = &module->import_functions[func_idx].u.function; + if (!func_ptr) { + snprintf(buf, sizeof(buf), + "failed to call unlinked import function (%s, %s)", + import_func->module_name, import_func->field_name); + wasm_set_exception(module_inst, buf); + return false; + } + + attachment = import_func->attachment; + if (import_func->call_conv_wasm_c_api) { + return wasm_runtime_invoke_c_api_native( + (WASMModuleInstanceCommon *)module_inst, func_ptr, func_type, argc, + argv, import_func->wasm_c_api_with_env, attachment); + } + else if (!import_func->call_conv_raw) { + signature = import_func->signature; + return wasm_runtime_invoke_native(exec_env, func_ptr, func_type, + signature, attachment, argv, argc, + argv); + } + else { + signature = import_func->signature; + return wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type, + signature, attachment, argv, argc, + argv); + } +} + +#if WASM_ENABLE_BULK_MEMORY != 0 +bool +llvm_jit_memory_init(WASMModuleInstance *module_inst, uint32 seg_index, + uint32 offset, uint32 len, uint32 dst) +{ + WASMMemoryInstance *memory_inst; + WASMModule *module; + uint8 *data = NULL; + uint8 *maddr; + uint64 seg_len = 0; + +#if WASM_ENABLE_JIT != 0 + if (Wasm_Module_AoT == module_inst->module_type) { + return aot_memory_init(module_inst, seg_index, offset, len, dst); + } +#endif + + memory_inst = wasm_get_default_memory(module_inst); + module = module_inst->module; + seg_len = module->data_segments[seg_index]->data_length; + data = module->data_segments[seg_index]->data; + + if (!wasm_runtime_validate_app_addr((WASMModuleInstanceCommon *)module_inst, + dst, len)) + return false; + + if ((uint64)offset + (uint64)len > seg_len) { + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; + } + + maddr = wasm_runtime_addr_app_to_native( + (WASMModuleInstanceCommon *)module_inst, dst); + + bh_memcpy_s(maddr, memory_inst->memory_data_size - dst, data + offset, len); + return true; +} + +bool +llvm_jit_data_drop(WASMModuleInstance *module_inst, uint32 seg_index) +{ +#if WASM_ENABLE_JIT != 0 + if (Wasm_Module_AoT == module_inst->module_type) { + return aot_data_drop(module_inst, seg_index); + } +#endif + + module_inst->module->data_segments[seg_index]->data_length = 0; + /* Currently we can't free the dropped data segment + as they are stored in wasm bytecode */ + return true; +} +#endif /* end of WASM_ENABLE_BULK_MEMORY != 0 */ + +#if WASM_ENABLE_REF_TYPES != 0 +void +llvm_jit_drop_table_seg(WASMModuleInstance *module_inst, uint32 tbl_seg_idx) +{ + WASMTableSeg *tbl_segs; + +#if WASM_ENABLE_JIT != 0 + if (Wasm_Module_AoT == module_inst->module_type) { + return aot_drop_table_seg(module_inst, tbl_seg_idx); + } +#endif + + tbl_segs = module_inst->module->table_segments; + tbl_segs[tbl_seg_idx].is_dropped = true; +} + +void +llvm_jit_table_init(WASMModuleInstance *module_inst, uint32 tbl_idx, + uint32 tbl_seg_idx, uint32 length, uint32 src_offset, + uint32 dst_offset) +{ + WASMTableInstance *tbl_inst; + WASMTableSeg *tbl_seg; + +#if WASM_ENABLE_JIT != 0 + if (Wasm_Module_AoT == module_inst->module_type) { + return aot_table_init(module_inst, tbl_idx, tbl_seg_idx, length, + src_offset, dst_offset); + } +#endif + + tbl_inst = wasm_get_table_inst(module_inst, tbl_idx); + tbl_seg = module_inst->module->table_segments + tbl_seg_idx; + + bh_assert(tbl_inst); + bh_assert(tbl_seg); + + if (!length) { + return; + } + + if (length + src_offset > tbl_seg->function_count + || dst_offset + length > tbl_inst->cur_size) { + jit_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS); + return; + } + + if (tbl_seg->is_dropped) { + jit_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS); + return; + } + + if (!wasm_elem_is_passive(tbl_seg->mode)) { + jit_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS); + return; + } + + bh_memcpy_s((uint8 *)tbl_inst + offsetof(WASMTableInstance, elems) + + dst_offset * sizeof(uint32), + (uint32)sizeof(uint32) * (tbl_inst->cur_size - dst_offset), + tbl_seg->func_indexes + src_offset, + (uint32)(length * sizeof(uint32))); +} + +void +llvm_jit_table_copy(WASMModuleInstance *module_inst, uint32 src_tbl_idx, + uint32 dst_tbl_idx, uint32 length, uint32 src_offset, + uint32 dst_offset) +{ + WASMTableInstance *src_tbl_inst; + WASMTableInstance *dst_tbl_inst; + +#if WASM_ENABLE_JIT != 0 + if (Wasm_Module_AoT == module_inst->module_type) { + aot_table_copy(module_inst, src_tbl_idx, dst_tbl_idx, length, + src_offset, dst_offset); + return; + } +#endif + + src_tbl_inst = wasm_get_table_inst(module_inst, src_tbl_idx); + dst_tbl_inst = wasm_get_table_inst(module_inst, dst_tbl_idx); + bh_assert(src_tbl_inst); + bh_assert(dst_tbl_inst); + + if ((uint64)dst_offset + length > dst_tbl_inst->cur_size + || (uint64)src_offset + length > src_tbl_inst->cur_size) { + jit_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS); + return; + } + + /* if src_offset >= dst_offset, copy from front to back */ + /* if src_offset < dst_offset, copy from back to front */ + /* merge all together */ + bh_memmove_s((uint8 *)dst_tbl_inst + offsetof(WASMTableInstance, elems) + + sizeof(uint32) * dst_offset, + (uint32)sizeof(uint32) * (dst_tbl_inst->cur_size - dst_offset), + (uint8 *)src_tbl_inst + offsetof(WASMTableInstance, elems) + + sizeof(uint32) * src_offset, + (uint32)sizeof(uint32) * length); +} + +void +llvm_jit_table_fill(WASMModuleInstance *module_inst, uint32 tbl_idx, + uint32 length, uint32 val, uint32 data_offset) +{ + WASMTableInstance *tbl_inst; + +#if WASM_ENABLE_JIT != 0 + if (Wasm_Module_AoT == module_inst->module_type) { + aot_table_fill(module_inst, tbl_idx, length, val, data_offset); + return; + } +#endif + + tbl_inst = wasm_get_table_inst(module_inst, tbl_idx); + bh_assert(tbl_inst); + + if (data_offset + length > tbl_inst->cur_size) { + jit_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS); + return; + } + + for (; length != 0; data_offset++, length--) { + tbl_inst->elems[data_offset] = val; + } +} + +uint32 +llvm_jit_table_grow(WASMModuleInstance *module_inst, uint32 tbl_idx, + uint32 inc_size, uint32 init_val) +{ + WASMTableInstance *tbl_inst; + uint32 i, orig_size, total_size; + +#if WASM_ENABLE_JIT != 0 + if (Wasm_Module_AoT == module_inst->module_type) { + return aot_table_grow(module_inst, tbl_idx, inc_size, init_val); + } +#endif + + tbl_inst = wasm_get_table_inst(module_inst, tbl_idx); + if (!tbl_inst) { + return (uint32)-1; + } + + orig_size = tbl_inst->cur_size; + + if (!inc_size) { + return orig_size; + } + + if (tbl_inst->cur_size > UINT32_MAX - inc_size) { /* integer overflow */ + return (uint32)-1; + } + + total_size = tbl_inst->cur_size + inc_size; + if (total_size > tbl_inst->max_size) { + return (uint32)-1; + } + + /* fill in */ + for (i = 0; i < inc_size; ++i) { + tbl_inst->elems[tbl_inst->cur_size + i] = init_val; + } + + tbl_inst->cur_size = total_size; + return orig_size; +} +#endif /* end of WASM_ENABLE_REF_TYPES != 0 */ + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 || WASM_ENABLE_PERF_PROFILING != 0 +bool +llvm_jit_alloc_frame(WASMExecEnv *exec_env, uint32 func_index) +{ + WASMModuleInstance *module_inst; + WASMInterpFrame *frame; + uint32 size; + +#if WASM_ENABLE_JIT != 0 + if (Wasm_Module_AoT == exec_env->module_inst->module_type) { + return aot_alloc_frame(exec_env, func_index); + } +#endif + + module_inst = (WASMModuleInstance *)exec_env->module_inst; + size = wasm_interp_interp_frame_size(0); + + frame = wasm_exec_env_alloc_wasm_frame(exec_env, size); + if (!frame) { + wasm_set_exception(module_inst, "wasm operand stack overflow"); + return false; + } + + frame->function = module_inst->e->functions + func_index; + frame->ip = NULL; + frame->sp = frame->lp; +#if WASM_ENABLE_PERF_PROFILING != 0 + frame->time_started = os_time_get_boot_microsecond(); +#endif + frame->prev_frame = wasm_exec_env_get_cur_frame(exec_env); + wasm_exec_env_set_cur_frame(exec_env, frame); + + return true; +} + +void +llvm_jit_free_frame(WASMExecEnv *exec_env) +{ + WASMInterpFrame *frame; + WASMInterpFrame *prev_frame; + +#if WASM_ENABLE_JIT != 0 + if (Wasm_Module_AoT == exec_env->module_inst->module_type) { + aot_free_frame(exec_env); + return; + } +#endif + + frame = wasm_exec_env_get_cur_frame(exec_env); + prev_frame = frame->prev_frame; + +#if WASM_ENABLE_PERF_PROFILING != 0 + if (frame->function) { + frame->function->total_exec_time += + os_time_get_boot_microsecond() - frame->time_started; + frame->function->total_exec_cnt++; + } +#endif + wasm_exec_env_free_wasm_frame(exec_env, frame); + wasm_exec_env_set_cur_frame(exec_env, prev_frame); +} +#endif /* end of WASM_ENABLE_DUMP_CALL_STACK != 0 \ + || WASM_ENABLE_PERF_PROFILING != 0 */ + +#endif /* end of WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 */ diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 4cdbf5ba..3146b18f 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -21,6 +21,53 @@ typedef struct WASMMemoryInstance WASMMemoryInstance; typedef struct WASMTableInstance WASMTableInstance; typedef struct WASMGlobalInstance WASMGlobalInstance; +/** + * When LLVM JIT, WAMR compiler or AOT is enabled, we should ensure that + * some offsets of the same field in the interpreter module instance and + * aot module instance are the same, so that the LLVM JITed/AOTed code + * can smoothly access the interpreter module instance. + * Same for the memory instance and table instance. + * We use the macro DefPointer to define some related pointer fields. + */ +#if (WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 \ + || WASM_ENABLE_AOT != 0) \ + && UINTPTR_MAX == UINT32_MAX +/* Add u32 padding if LLVM JIT, WAMR compiler or AOT is enabled on + 32-bit platform */ +#define DefPointer(type, field) \ + type field; \ + uint32 field##_padding +#else +#define DefPointer(type, field) type field +#endif + +typedef enum WASMExceptionID { + EXCE_UNREACHABLE = 0, + EXCE_OUT_OF_MEMORY, + EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, + EXCE_INTEGER_OVERFLOW, + EXCE_INTEGER_DIVIDE_BY_ZERO, + EXCE_INVALID_CONVERSION_TO_INTEGER, + EXCE_INVALID_FUNCTION_TYPE_INDEX, + EXCE_INVALID_FUNCTION_INDEX, + EXCE_UNDEFINED_ELEMENT, + EXCE_UNINITIALIZED_ELEMENT, + EXCE_CALL_UNLINKED_IMPORT_FUNC, + EXCE_NATIVE_STACK_OVERFLOW, + EXCE_UNALIGNED_ATOMIC, + EXCE_AUX_STACK_OVERFLOW, + EXCE_AUX_STACK_UNDERFLOW, + EXCE_OUT_OF_BOUNDS_TABLE_ACCESS, + EXCE_OPERAND_STACK_OVERFLOW, + EXCE_ALREADY_THROWN, + EXCE_NUM, +} WASMExceptionID; + +typedef union { + uint64 u64; + uint32 u32[2]; +} MemBound; + struct WASMMemoryInstance { /* Module type */ uint32 module_type; @@ -35,59 +82,40 @@ struct WASMMemoryInstance { uint32 max_page_count; /* Memory data size */ uint32 memory_data_size; - /** * Memory data begin address, Note: * the app-heap might be inserted in to the linear memory, * when memory is re-allocated, the heap data and memory data * must be copied to new memory also */ - uint8 *memory_data; + DefPointer(uint8 *, memory_data); /* Memory data end address */ - uint8 *memory_data_end; + DefPointer(uint8 *, memory_data_end); /* Heap data base address */ - uint8 *heap_data; + DefPointer(uint8 *, heap_data); /* Heap data end address */ - uint8 *heap_data_end; + DefPointer(uint8 *, heap_data_end); /* The heap created */ - void *heap_handle; + DefPointer(void *, heap_handle); -#if WASM_ENABLE_SHARED_MEMORY != 0 - /* mutex lock for the memory, used in atomic operation */ - korp_mutex mem_lock; -#endif - -#if WASM_ENABLE_FAST_JIT != 0 -#if UINTPTR_MAX == UINT64_MAX - uint64 mem_bound_check_1byte; - uint64 mem_bound_check_2bytes; - uint64 mem_bound_check_4bytes; - uint64 mem_bound_check_8bytes; - uint64 mem_bound_check_16bytes; -#else - uint32 mem_bound_check_1byte; - uint32 mem_bound_check_2bytes; - uint32 mem_bound_check_4bytes; - uint32 mem_bound_check_8bytes; - uint32 mem_bound_check_16bytes; -#endif +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_AOT != 0 + MemBound mem_bound_check_1byte; + MemBound mem_bound_check_2bytes; + MemBound mem_bound_check_4bytes; + MemBound mem_bound_check_8bytes; + MemBound mem_bound_check_16bytes; #endif }; struct WASMTableInstance { - /* The element type, VALUE_TYPE_FUNCREF/EXTERNREF currently */ - uint8 elem_type; /* Current size */ uint32 cur_size; /* Maximum size */ uint32 max_size; -#if WASM_ENABLE_MULTI_MODULE != 0 - /* just for import, keep the reference here */ - WASMTableInstance *table_inst_linked; -#endif - /* Base address */ - uint8 base_addr[1]; + /* Table elements */ + uint32 elems[1]; }; struct WASMGlobalInstance { @@ -149,7 +177,6 @@ typedef struct WASMExportFuncInstance { WASMFunctionInstance *function; } WASMExportFuncInstance; -#if WASM_ENABLE_MULTI_MODULE != 0 typedef struct WASMExportGlobInstance { char *name; WASMGlobalInstance *global; @@ -164,8 +191,40 @@ typedef struct WASMExportMemInstance { char *name; WASMMemoryInstance *memory; } WASMExportMemInstance; + +/* Extra info of WASM module instance for interpreter/jit mode */ +typedef struct WASMModuleInstanceExtra { + WASMGlobalInstance *globals; + WASMFunctionInstance *functions; + + uint32 global_count; + uint32 function_count; + + WASMFunctionInstance *start_function; + WASMFunctionInstance *malloc_function; + WASMFunctionInstance *free_function; + WASMFunctionInstance *retain_function; + +#if WASM_ENABLE_SHARED_MEMORY != 0 + /* lock for shared memory atomic operations */ + korp_mutex mem_lock; + bool mem_lock_inited; #endif +#if WASM_ENABLE_MULTI_MODULE != 0 + bh_list sub_module_inst_list_head; + bh_list *sub_module_inst_list; + /* linked table instances of import table instances */ + WASMTableInstance **table_insts_linked; +#endif + +#if WASM_ENABLE_MEMORY_PROFILING != 0 + uint32 max_aux_stack_used; +#endif +} WASMModuleInstanceExtra; + +struct AOTFuncPerfProfInfo; + struct WASMModuleInstance { /* Module instance type, for module instance loaded from WASM bytecode binary, this field is Wasm_Module_Bytecode; @@ -175,78 +234,82 @@ struct WASMModuleInstance { uint32 module_type; uint32 memory_count; + DefPointer(WASMMemoryInstance **, memories); + + /* global and table info */ + uint32 global_data_size; uint32 table_count; - uint32 global_count; - uint32 function_count; + DefPointer(uint8 *, global_data); + /* For AOTModuleInstance, it denotes `AOTTableInstance *` */ + DefPointer(WASMTableInstance **, tables); + + /* import func ptrs + llvm jit func ptrs */ + DefPointer(void **, func_ptrs); + + /* function type indexes */ + DefPointer(uint32 *, func_type_indexes); uint32 export_func_count; -#if WASM_ENABLE_MULTI_MODULE != 0 - uint32 export_glob_count; - uint32 export_mem_count; - uint32 export_tab_count; -#endif - - /* Array of function pointers to import functions */ - void **import_func_ptrs; -#if WASM_ENABLE_FAST_JIT != 0 - /* point to JITed functions */ - void **fast_jit_func_ptrs; - uint32 *func_type_indexes; -#endif - - WASMMemoryInstance **memories; - WASMTableInstance **tables; - WASMGlobalInstance *globals; - WASMFunctionInstance *functions; - - WASMExportFuncInstance *export_functions; -#if WASM_ENABLE_MULTI_MODULE != 0 - WASMExportGlobInstance *export_globals; - WASMExportMemInstance *export_memories; - WASMExportTabInstance *export_tables; -#endif - - WASMMemoryInstance *default_memory; - WASMTableInstance *default_table; - /* Global data of global instances */ - uint8 *global_data; - - WASMFunctionInstance *start_function; - WASMFunctionInstance *malloc_function; - WASMFunctionInstance *free_function; - WASMFunctionInstance *retain_function; - - WASMModule *module; - -#if WASM_ENABLE_LIBC_WASI != 0 - WASIContext *wasi_ctx; -#endif - - WASMExecEnv *exec_env_singleton; - - /* Default WASM stack size of threads of this Module instance. */ - uint32 default_wasm_stack_size; + uint32 export_global_count; + uint32 export_memory_count; + uint32 export_table_count; + /* For AOTModuleInstance, it denotes `AOTFunctionInstance *` */ + DefPointer(WASMExportFuncInstance *, export_functions); + DefPointer(WASMExportGlobInstance *, export_globals); + DefPointer(WASMExportMemInstance *, export_memories); + DefPointer(WASMExportTabInstance *, export_tables); /* The exception buffer of wasm interpreter for current thread. */ char cur_exception[128]; -#if WASM_ENABLE_DUMP_CALL_STACK != 0 - Vector *frames; -#endif + /* The WASM module or AOT module, for AOTModuleInstance, + it denotes `AOTModule *` */ + DefPointer(WASMModule *, module); - /* The custom data that can be set/get by - * wasm_set_custom_data/wasm_get_custom_data */ - void *custom_data; - -#if WASM_ENABLE_MULTI_MODULE != 0 - /* TODO: add mutex for mutli-threads? */ - bh_list sub_module_inst_list_head; - bh_list *sub_module_inst_list; +#if WASM_ENABLE_LIBC_WASI + /* WASI context */ + DefPointer(WASIContext *, wasi_ctx); +#else + DefPointer(void *, wasi_ctx); #endif + DefPointer(WASMExecEnv *, exec_env_singleton); + /* Array of function pointers to import functions, + not available in AOTModuleInstance */ + DefPointer(void **, import_func_ptrs); + /* Array of function pointers to fast jit functions, + not available in AOTModuleInstance */ + DefPointer(void **, fast_jit_func_ptrs); + /* The custom data that can be set/get by wasm_{get|set}_custom_data */ + DefPointer(void *, custom_data); + /* Stack frames, used in call stack dump and perf profiling */ + DefPointer(Vector *, frames); + /* Function performance profiling info list, only available + in AOTModuleInstance */ + DefPointer(struct AOTFuncPerfProfInfo *, func_perf_profilings); + /* WASM/AOT module extra info, for AOTModuleInstance, + it denotes `AOTModuleInstanceExtra *` */ + DefPointer(WASMModuleInstanceExtra *, e); -#if WASM_ENABLE_MEMORY_PROFILING != 0 - uint32 max_aux_stack_used; -#endif + /* Default WASM operand stack size */ + uint32 default_wasm_stack_size; + uint32 reserved[3]; + + /* + * +------------------------------+ <-- memories + * | WASMMemoryInstance[mem_count], mem_count is always 1 for LLVM JIT/AOT + * +------------------------------+ <-- global_data + * | global data + * +------------------------------+ <-- tables + * | WASMTableInstance[table_count] + * +------------------------------+ <-- e + * | WASMModuleInstanceExtra + * +------------------------------+ + */ + union { + uint64 _make_it_8_byte_aligned_; + WASMMemoryInstance memory_instances[1]; + uint8 bytes[1]; + } global_table_data; }; struct WASMInterpFrame; @@ -342,12 +405,12 @@ wasm_create_exec_env_and_call_function(WASMModuleInstance *module_inst, WASMFunctionInstance *function, unsigned argc, uint32 argv[]); -bool -wasm_create_exec_env_singleton(WASMModuleInstance *module_inst); - void wasm_set_exception(WASMModuleInstance *module, const char *exception); +void +wasm_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id); + const char * wasm_get_exception(WASMModuleInstance *module); @@ -366,45 +429,25 @@ uint32 wasm_module_dup_data(WASMModuleInstance *module_inst, const char *src, uint32 size); +/** + * Check whether the app address and the buf is inside the linear memory, + * and convert the app address into native address + */ bool -wasm_validate_app_addr(WASMModuleInstance *module_inst, uint32 app_offset, - uint32 size); +wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, + uint32 app_buf_addr, uint32 app_buf_size, + void **p_native_addr); + +WASMMemoryInstance * +wasm_get_default_memory(WASMModuleInstance *module_inst); bool -wasm_validate_app_str_addr(WASMModuleInstance *module_inst, uint32 app_offset); - -bool -wasm_validate_native_addr(WASMModuleInstance *module_inst, void *native_ptr, - uint32 size); - -void * -wasm_addr_app_to_native(WASMModuleInstance *module_inst, uint32 app_offset); - -uint32 -wasm_addr_native_to_app(WASMModuleInstance *module_inst, void *native_ptr); - -bool -wasm_get_app_addr_range(WASMModuleInstance *module_inst, uint32 app_offset, - uint32 *p_app_start_offset, uint32 *p_app_end_offset); - -bool -wasm_get_native_addr_range(WASMModuleInstance *module_inst, uint8 *native_ptr, - uint8 **p_native_start_addr, - uint8 **p_native_end_addr); - -bool -wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count); +wasm_enlarge_memory(WASMModuleInstance *module_inst, uint32 inc_page_count); bool wasm_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, uint32 argc, uint32 argv[]); -#if WASM_ENABLE_FAST_JIT != 0 -bool -jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, - uint32 type_idx, uint32 argc, uint32 argv[]); -#endif - #if WASM_ENABLE_THREAD_MGR != 0 bool wasm_set_aux_stack(WASMExecEnv *exec_env, uint32 start_offset, uint32 size); @@ -413,16 +456,6 @@ bool wasm_get_aux_stack(WASMExecEnv *exec_env, uint32 *start_offset, uint32 *size); #endif -#ifdef OS_ENABLE_HW_BOUND_CHECK -#ifndef BH_PLATFORM_WINDOWS -void -wasm_signal_handler(WASMSignalInfo *sig_info); -#else -LONG -wasm_exception_handler(WASMSignalInfo *sig_info); -#endif -#endif - void wasm_get_module_mem_consumption(const WASMModule *module, WASMModuleMemConsumption *mem_conspn); @@ -456,13 +489,14 @@ wasm_enlarge_table(WASMModuleInstance *module_inst, uint32 table_idx, #endif /* WASM_ENABLE_REF_TYPES != 0 */ static inline WASMTableInstance * -wasm_get_table_inst(const WASMModuleInstance *module_inst, const uint32 tbl_idx) +wasm_get_table_inst(const WASMModuleInstance *module_inst, uint32 tbl_idx) { /* careful, it might be a table in another module */ WASMTableInstance *tbl_inst = module_inst->tables[tbl_idx]; #if WASM_ENABLE_MULTI_MODULE != 0 - if (tbl_inst->table_inst_linked) { - tbl_inst = tbl_inst->table_inst_linked; + if (tbl_idx < module_inst->module->import_table_count + && module_inst->e->table_insts_linked[tbl_idx]) { + tbl_inst = module_inst->e->table_insts_linked[tbl_idx]; } #endif bh_assert(tbl_inst); @@ -495,6 +529,82 @@ const uint8 * wasm_loader_get_custom_section(WASMModule *module, const char *name, uint32 *len); +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 +void +jit_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id); + +/** + * Check whether the app address and the buf is inside the linear memory, + * and convert the app address into native address + */ +bool +jit_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, + uint32 app_buf_addr, uint32 app_buf_size, + void **p_native_addr); +#endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 */ + +#if WASM_ENABLE_FAST_JIT != 0 +bool +fast_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 type_idx, uint32 argc, uint32 *argv); + +bool +fast_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, + struct WASMInterpFrame *prev_frame); +#endif + +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 +bool +llvm_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 argc, uint32 *argv); + +bool +llvm_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc, + uint32 *argv); + +#if WASM_ENABLE_BULK_MEMORY != 0 +bool +llvm_jit_memory_init(WASMModuleInstance *module_inst, uint32 seg_index, + uint32 offset, uint32 len, uint32 dst); + +bool +llvm_jit_data_drop(WASMModuleInstance *module_inst, uint32 seg_index); +#endif + +#if WASM_ENABLE_REF_TYPES != 0 +void +llvm_jit_drop_table_seg(WASMModuleInstance *module_inst, uint32 tbl_seg_idx); + +void +llvm_jit_table_init(WASMModuleInstance *module_inst, uint32 tbl_idx, + uint32 tbl_seg_idx, uint32 length, uint32 src_offset, + uint32 dst_offset); + +void +llvm_jit_table_copy(WASMModuleInstance *module_inst, uint32 src_tbl_idx, + uint32 dst_tbl_idx, uint32 length, uint32 src_offset, + uint32 dst_offset); + +void +llvm_jit_table_fill(WASMModuleInstance *module_inst, uint32 tbl_idx, + uint32 length, uint32 val, uint32 data_offset); + +uint32 +llvm_jit_table_grow(WASMModuleInstance *module_inst, uint32 tbl_idx, + uint32 inc_entries, uint32 init_val); +#endif + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 || WASM_ENABLE_PERF_PROFILING != 0 +bool +llvm_jit_alloc_frame(WASMExecEnv *exec_env, uint32 func_index); + +void +llvm_jit_free_frame(WASMExecEnv *exec_env); +#endif +#endif /* end of WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 */ + #ifdef __cplusplus } #endif diff --git a/core/iwasm/libraries/debug-engine/debug_engine.c b/core/iwasm/libraries/debug-engine/debug_engine.c index e62ba5a2..10e39b57 100644 --- a/core/iwasm/libraries/debug-engine/debug_engine.c +++ b/core/iwasm/libraries/debug-engine/debug_engine.c @@ -617,7 +617,7 @@ wasm_debug_instance_get_memregion(WASMDebugInstance *instance, uint64 addr) break; case WasmMemory: { - memory = module_inst->default_memory; + memory = wasm_get_default_memory(module_inst); if (memory) { num_bytes_per_page = memory->num_bytes_per_page; @@ -715,7 +715,7 @@ wasm_debug_instance_get_linear_mem(WASMDebugInstance *instance, uint64 offset, return false; module_inst = (WASMModuleInstance *)exec_env->module_inst; - memory = module_inst->default_memory; + memory = wasm_get_default_memory(module_inst); if (memory) { num_bytes_per_page = memory->num_bytes_per_page; linear_mem_size = num_bytes_per_page * memory->cur_page_count; @@ -748,7 +748,7 @@ wasm_debug_instance_set_linear_mem(WASMDebugInstance *instance, uint64 offset, return false; module_inst = (WASMModuleInstance *)exec_env->module_inst; - memory = module_inst->default_memory; + memory = wasm_get_default_memory(module_inst); if (memory) { num_bytes_per_page = memory->num_bytes_per_page; linear_mem_size = num_bytes_per_page * memory->cur_page_count; @@ -1134,10 +1134,10 @@ wasm_debug_instance_get_global(WASMDebugInstance *instance, int32 frame_index, module_inst = (WASMModuleInstance *)exec_env->module_inst; global_data = module_inst->global_data; - globals = module_inst->globals; + globals = module_inst->e->globals; if ((global_index < 0) - || ((uint32)global_index >= module_inst->global_count)) { + || ((uint32)global_index >= module_inst->e->global_count)) { return false; } global = globals + global_index; diff --git a/product-mini/platforms/android/CMakeLists.txt b/product-mini/platforms/android/CMakeLists.txt index a65b1808..638b6ab0 100644 --- a/product-mini/platforms/android/CMakeLists.txt +++ b/product-mini/platforms/android/CMakeLists.txt @@ -23,7 +23,8 @@ set (WAMR_BUILD_PLATFORM "android") set (WAMR_BUILD_TARGET "X86_32") set (WAMR_BUILD_TYPE Release) set (WAMR_BUILD_INTERP 1) -set (WAMR_BUILD_AOT 0) +set (WAMR_BUILD_AOT 1) +set (WAMR_BUILD_JIT 0) set (WAMR_BUILD_LIBC_BUILTIN 1) set (WAMR_BUILD_LIBC_WASI 1) diff --git a/samples/wasm-c-api/CMakeLists.txt b/samples/wasm-c-api/CMakeLists.txt index 63fb06e7..05b910a5 100644 --- a/samples/wasm-c-api/CMakeLists.txt +++ b/samples/wasm-c-api/CMakeLists.txt @@ -125,6 +125,7 @@ set(MM_UTIL src/utils/multi_module_utils.c) set(EXAMPLES callback callback_chain + clone empty_imports global hello @@ -135,6 +136,12 @@ set(EXAMPLES trap ) +if(WAMR_BUILD_JIT AND WAMR_BUILD_LAZY_JIT) + if((${WAMR_BUILD_JIT} EQUAL 1) AND (${WAMR_BUILD_LAZY_JIT} EQUAL 1)) + list(APPEND EXAMPLES serialize) + endif() +endif() + foreach(EX ${EXAMPLES}) set(SRC ${CMAKE_CURRENT_LIST_DIR}/src/${EX}.c) diff --git a/samples/wasm-c-api/src/clone.c b/samples/wasm-c-api/src/clone.c new file mode 100644 index 00000000..c27dc680 --- /dev/null +++ b/samples/wasm-c-api/src/clone.c @@ -0,0 +1,518 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include + +#include "wasm_c_api.h" + +#define WORKER_NUMBER 10 + +/******************************* VM *******************************/ +/* Use wasm_vm_t and vm_xxx to simulate a minimal Wasm VM in Envoy */ + +typedef struct _vm { + wasm_engine_t *engine; + wasm_store_t *store; + wasm_module_t *module; + wasm_instance_t *instance; + wasm_func_t **function_list; + wasm_memory_t *memory; + wasm_table_t *table; + wasm_extern_vec_t *exports; +} wasm_vm_t; + +typedef enum _clone_level { + not_cloneable = 0, + compiled_bytecode, + instantiated_module +} clone_level; + +typedef struct _thread_arg_t { + char name[32]; + bool *ready_go_flag; + pthread_mutex_t *ready_go_lock; + pthread_cond_t *ready_go_cond; + const wasm_vm_t *base_vm; +} thread_arg_t; + +wasm_vm_t * +vm_new() +{ + wasm_vm_t *vm = NULL; + + vm = malloc(sizeof(struct _vm)); + if (!vm) + goto fail; + + memset(vm, 0, sizeof(wasm_vm_t)); + + vm->engine = wasm_engine_new(); + if (!vm->engine) + goto fail; + + vm->store = wasm_store_new(vm->engine); + if (!vm->store) + goto fail; + + return vm; + +fail: + if (vm) { + if (vm->engine) + wasm_engine_delete(vm->engine); + + free(vm); + } + return NULL; +} + +bool +vm_load(wasm_vm_t *vm, const wasm_byte_vec_t *binary) +{ + vm->module = wasm_module_new(vm->store, binary); + return vm->module != NULL; +} + +bool +vm_link(wasm_vm_t *vm, wasm_extern_vec_t *imports) +{ + vm->instance = wasm_instance_new(vm->store, vm->module, imports, NULL); + if (!vm->instance) + goto fail; + + vm->exports = malloc(sizeof(wasm_extern_vec_t)); + if (!vm->exports) + goto fail; + + memset(vm->exports, 0, sizeof(wasm_extern_vec_t)); + wasm_instance_exports(vm->instance, vm->exports); + /* an exported memory, and two exported functions */ + assert(vm->exports->size == 3); + + /* bind memory */ + assert(wasm_extern_kind(vm->exports->data[0]) == WASM_EXTERN_MEMORY); + vm->memory = wasm_extern_as_memory(vm->exports->data[0]); + + vm->function_list = malloc(2 * sizeof(wasm_func_t *)); + if (!vm->function_list) + goto fail; + + memset(vm->function_list, 0, sizeof(2 * sizeof(wasm_func_t *))); + + /* bind wasm_set_byte(...) */ + assert(wasm_extern_kind(vm->exports->data[1]) == WASM_EXTERN_FUNC); + vm->function_list[0] = wasm_extern_as_func(vm->exports->data[1]); + + /* bind wasm_get_byte(...) */ + assert(wasm_extern_kind(vm->exports->data[2]) == WASM_EXTERN_FUNC); + vm->function_list[1] = wasm_extern_as_func(vm->exports->data[2]); + + return true; +fail: + return false; +} + +wasm_vm_t * +vm_clone_from_module(const wasm_vm_t *base) +{ + printf("Initializing...\n"); + wasm_vm_t *secondary = NULL; + + secondary = vm_new(); + if (secondary) { + printf("Reuse module and bypass vm_load()..."); + secondary->module = base->module; + } + + return secondary; +} + +wasm_vm_t * +vm_clone_from_instance(const wasm_vm_t *base) +{ + /** + * if do a clone of the level instantiated_module, need to malloc and + * initialie + * - global. WASMGlobalIntance and global data + * - memory. WAAMMemoryInstance, memory_data and heap + * - table. WASMTableInstance, table_data + * - exports. all global, memory and table + * + * it is almost everything in wasm_instantiate() except funciton. + */ + (void)base; + printf("Unsupported\n"); + return NULL; +} + +wasm_vm_t * +vm_clone(const wasm_vm_t *base, clone_level level) +{ + if (level == not_cloneable) + return NULL; + + if (level == compiled_bytecode) + return vm_clone_from_module(base); + else + return vm_clone_from_instance(base); +} + +wasm_vm_t * +vm_release(wasm_vm_t *vm) +{ + if (!vm) + return NULL; + + if (vm->function_list) { + free(vm->function_list); + vm->function_list = NULL; + } + + if (vm->exports) { + wasm_extern_vec_delete(vm->exports); + free(vm->exports); + vm->exports = NULL; + } + + wasm_instance_delete(vm->instance); + vm->instance = NULL; + + wasm_module_delete(vm->module); + vm->module = NULL; + + wasm_store_delete(vm->store); + vm->store = NULL; + + wasm_engine_delete(vm->engine); + vm->engine = NULL; + + free(vm); + return NULL; +} + +bool +vm_memory_set_byte(const wasm_vm_t *vm, uint32_t offset, uint8_t byte) +{ + byte_t *data = wasm_memory_data(vm->memory); + assert(data); + *(data + offset) = byte; + return true; +} + +bool +vm_memory_get_byte(const wasm_vm_t *vm, uint32_t offset, uint8_t *byte) +{ + byte_t *data = wasm_memory_data(vm->memory); + assert(data); + *byte = *(data + offset); + return true; +} + +bool +vm_function_set_byte(const wasm_vm_t *vm, uint32_t offset, uint8_t byte) +{ + wasm_val_t a_v[2] = { WASM_I32_VAL(offset), WASM_I32_VAL(byte) }; + wasm_val_vec_t args = WASM_ARRAY_VEC(a_v); + wasm_val_vec_t results = WASM_EMPTY_VEC; + wasm_trap_t *trap = wasm_func_call(vm->function_list[0], &args, &results); + if (trap) + printf("call wasm_set_byte failed"); + + return trap != NULL; +} + +bool +vm_function_get_byte(const wasm_vm_t *vm, uint32_t offset, uint8_t *byte) +{ + wasm_val_t a_v[1] = { WASM_I32_VAL(offset) }; + wasm_val_vec_t args = WASM_ARRAY_VEC(a_v); + wasm_val_t r_v[1] = { WASM_INIT_VAL }; + wasm_val_vec_t results = WASM_ARRAY_VEC(r_v); + wasm_trap_t *trap = wasm_func_call(vm->function_list[1], &args, &results); + if (trap) { + printf("call wasm_get_byte failed"); + return false; + } + + assert(results.data->kind == WASM_I32); + *byte = results.data->of.i32; + return true; +} + +static bool +load_wasm_file_content(const char *file_name, wasm_byte_vec_t *out) +{ + bool ret = false; +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_INTERP == 0 + FILE *file = fopen(file_name, "rb"); +#else + FILE *file = fopen(file_name, "rb"); +#endif + if (!file) { + printf("> Error loading .wasm!\n"); + goto quit; + } + + int offset = fseek(file, 0L, SEEK_END); + if (offset == -1) { + printf("> Error loading .wasm!\n"); + goto close_file; + } + + long file_size = ftell(file); + if (file_size == -1) { + printf("> Error loading .wasm!\n"); + goto close_file; + } + + offset = fseek(file, 0L, SEEK_SET); + if (offset == -1) { + printf("> Error loading .wasm!\n"); + goto close_file; + } + + wasm_byte_vec_new_uninitialized(out, file_size); + if (fread(out->data, file_size, 1, file) != 1) { + printf("> Error loading content!\n"); + goto close_file; + } + + ret = true; +close_file: + fclose(file); +quit: + return ret; +} + +static pthread_key_t name_key; + +wasm_trap_t * +report_cb(const wasm_val_vec_t *args, wasm_val_vec_t *results) +{ + (void)results; + + assert(args->data[0].kind == WASM_I32); + uint32_t chk_pnt_no = args->data[0].of.i32; + + char *name = pthread_getspecific(name_key); + printf("[%s] Pass CHK POINT #%u\n", name, chk_pnt_no); + + return NULL; +} + +bool +run_code_start(wasm_vm_t **out) +{ + bool ret = false; + + printf("Initializing...\n"); + wasm_vm_t *vm = vm_new(); + if (!vm) + goto fail; + + printf("Loading binary...\n"); + wasm_byte_vec_t binary = { 0 }; +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_INTERP == 0 + const char *file_name = "clone.aot"; +#else + const char *file_name = "clone.wasm"; +#endif + if (!load_wasm_file_content(file_name, &binary)) + goto release_vm; + + printf("Compiling module...\n"); + ret = vm_load(vm, &binary); + wasm_byte_vec_delete(&binary); + if (!ret) + goto release_vm; + + printf("Creating callback...\n"); + wasm_functype_t *callback_type = + wasm_functype_new_1_0(wasm_valtype_new_i32()); + if (!callback_type) + goto release_vm; + + wasm_func_t *callback = wasm_func_new(vm->store, callback_type, report_cb); + wasm_functype_delete(callback_type); + if (!callback) + goto release_vm; + + printf("Instantiating module...\n"); + wasm_extern_t *externs[] = { wasm_func_as_extern(callback) }; + wasm_extern_vec_t imports = WASM_ARRAY_VEC(externs); + ret = vm_link(vm, &imports); + wasm_func_delete(callback); + if (!ret) + goto release_vm; + + *out = vm; + return true; + +release_vm: + vm_release(vm); +fail: + return false; +} + +bool +run_warm_start_w_compiled_bytecode(const wasm_vm_t *first, wasm_vm_t **out) +{ + bool ret; + wasm_vm_t *secondary = vm_clone(first, compiled_bytecode); + if (!secondary) + goto fail; + + printf("Creating callback...\n"); + wasm_functype_t *callback_type = + wasm_functype_new_1_0(wasm_valtype_new_i32()); + if (!callback_type) + goto release_vm; + + wasm_func_t *callback = + wasm_func_new(secondary->store, callback_type, report_cb); + wasm_functype_delete(callback_type); + if (!callback) + goto release_vm; + + printf("Instantiating module...\n"); + wasm_extern_t *externs[] = { wasm_func_as_extern(callback) }; + wasm_extern_vec_t imports = WASM_ARRAY_VEC(externs); + ret = vm_link(secondary, &imports); + wasm_func_delete(callback); + if (!ret) + goto release_vm; + + *out = secondary; + return true; + +release_vm: + vm_release(secondary); +fail: + return false; +} + +bool +run_warm_start_w_instantiated_module(const wasm_vm_t *first, wasm_vm_t **out) +{ + wasm_vm_t *secondary = vm_clone(first, instantiated_module); + if (!secondary) + return false; + + *out = secondary; + return true; +} + +void +run_test(const wasm_vm_t *vm) +{ + uint8_t byte = 0xFF; + + /* read initialization */ + vm_function_get_byte(vm, 10, &byte); + assert(byte == 0x0); + vm_memory_get_byte(vm, 10, &byte); + assert(byte == 0x0); + + /* read after writing */ + vm_function_set_byte(vm, 16, 0xab); + vm_function_get_byte(vm, 16, &byte); + assert(byte == 0xab); + + vm_memory_set_byte(vm, 16, 0xcd); + vm_memory_get_byte(vm, 16, &byte); + assert(byte == 0xcd); + + /* reading and writing across */ + vm_function_set_byte(vm, 16, 0xef); + vm_memory_get_byte(vm, 16, &byte); + assert(byte == 0xef); + + vm_memory_set_byte(vm, 16, 0x67); + vm_function_get_byte(vm, 16, &byte); + assert(byte == 0x67); + + printf("All Passed ...\n"); +} + +static void * +thrd_func(void *arg) +{ + thread_arg_t *thrd_arg = (thread_arg_t *)arg; + printf("Running warm start at %s...\n", thrd_arg->name); + + pthread_setspecific(name_key, thrd_arg->name); + + wasm_vm_t *vm; + if (!run_warm_start_w_compiled_bytecode(thrd_arg->base_vm, &vm)) + return NULL; + + pthread_mutex_trylock(thrd_arg->ready_go_lock); + while (!(*thrd_arg->ready_go_flag)) { + pthread_cond_wait(thrd_arg->ready_go_cond, thrd_arg->ready_go_lock); + } + pthread_mutex_unlock(thrd_arg->ready_go_lock); + + printf("Running test at %s...\n", thrd_arg->name); + run_test(vm); + + vm_release(vm); + pthread_exit(NULL); + return NULL; +} + +int +main() +{ + int ret = EXIT_FAILURE; + bool ready_go_flag = false; + pthread_mutex_t ready_go_lock = PTHREAD_MUTEX_INITIALIZER; + pthread_cond_t ready_go_cond = PTHREAD_COND_INITIALIZER; + pthread_key_create(&name_key, NULL); + pthread_setspecific(name_key, "Execution Thread"); + + printf("Running cold start at the execution thread...\n"); + wasm_vm_t *base_vm; + if (!run_code_start(&base_vm)) + goto quit; + run_test(base_vm); + + printf("Running warm start at other threads...\n"); + pthread_mutex_trylock(&ready_go_lock); + + pthread_t tids[WORKER_NUMBER] = { 0 }; + thread_arg_t thrd_args[WORKER_NUMBER] = { 0 }; + for (size_t i = 0; i < sizeof(tids) / sizeof(tids[0]); i++) { + thread_arg_t *thrd_arg = thrd_args + i; + + snprintf(thrd_arg->name, 32, "Worker#%lu", i); + thrd_arg->ready_go_cond = &ready_go_cond; + thrd_arg->ready_go_lock = &ready_go_lock; + thrd_arg->ready_go_flag = &ready_go_flag; + thrd_arg->base_vm = base_vm; + + int ret = pthread_create(&tids[i], NULL, thrd_func, thrd_arg); + if (ret != 0) + break; + } + + sleep(5); + + ready_go_flag = true; + pthread_mutex_unlock(&ready_go_lock); + pthread_cond_broadcast(&ready_go_cond); + + for (size_t i = 0; i < sizeof(tids) / sizeof(tids[0]); i++) { + if (tids[i] != 0) + pthread_join(tids[i], NULL); + } + vm_release(base_vm); + ret = EXIT_SUCCESS; +quit: + return ret; +} diff --git a/samples/wasm-c-api/src/clone.wat b/samples/wasm-c-api/src/clone.wat new file mode 100644 index 00000000..e9934cc0 --- /dev/null +++ b/samples/wasm-c-api/src/clone.wat @@ -0,0 +1,15 @@ +(module + (func $report (import "" "report") (param i32)) + + (memory (export "mem") 1 1) + + (func $wasm_set_byte (export "set_byte") (param i32 i32) + (call $report (i32.const 1)) + (i32.store8 (local.get 0) (local.get 1)) + ) + + (func $wasm_get_byte (export "get_byte") (param i32) (result i32) + (call $report (i32.const 2)) + (i32.load(local.get 0)) + ) +) \ No newline at end of file diff --git a/samples/wasm-c-api/src/serialize.c b/samples/wasm-c-api/src/serialize.c new file mode 100644 index 00000000..d6570691 --- /dev/null +++ b/samples/wasm-c-api/src/serialize.c @@ -0,0 +1,129 @@ +#include +#include +#include +#include + +#include "wasm_c_api.h" + +#define own + +// A function to be called from Wasm code. +own wasm_trap_t * +hello_callback(const wasm_val_vec_t *args, wasm_val_vec_t *results) +{ + printf("Calling back...\n"); + printf("> Hello World!\n"); + return NULL; +} + +int +main(int argc, const char *argv[]) +{ + // Initialize. + printf("Initializing...\n"); + wasm_engine_t *engine = wasm_engine_new(); + wasm_store_t *store = wasm_store_new(engine); + + // Load binary. + printf("Loading binary...\n"); + FILE *file = fopen("serialize.wasm", "rb"); + if (!file) { + printf("> Error loading module!\n"); + return 1; + } + fseek(file, 0L, SEEK_END); + size_t file_size = ftell(file); + fseek(file, 0L, SEEK_SET); + wasm_byte_vec_t binary; + wasm_byte_vec_new_uninitialized(&binary, file_size); + if (fread(binary.data, file_size, 1, file) != 1) { + printf("> Error loading module!\n"); + return 1; + } + fclose(file); + + // Compile. + printf("Compiling module...\n"); + own wasm_module_t *module = wasm_module_new(store, &binary); + if (!module) { + printf("> Error compiling module!\n"); + return 1; + } + + wasm_byte_vec_delete(&binary); + + // Serialize module. + printf("Serializing module...\n"); + own wasm_byte_vec_t serialized; + wasm_module_serialize(module, &serialized); + + wasm_module_delete(module); + + // Deserialize module. + printf("Deserializing module...\n"); + own wasm_module_t *deserialized = + wasm_module_deserialize(store, &serialized); + if (!deserialized) { + printf("> Error deserializing module!\n"); + return 1; + } + + wasm_byte_vec_delete(&serialized); + + // Create external print functions. + printf("Creating callback...\n"); + own wasm_functype_t *hello_type = wasm_functype_new_0_0(); + own wasm_func_t *hello_func = + wasm_func_new(store, hello_type, hello_callback); + + wasm_functype_delete(hello_type); + + // Instantiate. + printf("Instantiating deserialized module...\n"); + wasm_extern_t *externs[] = { wasm_func_as_extern(hello_func) }; + wasm_extern_vec_t imports = WASM_ARRAY_VEC(externs); + own wasm_instance_t *instance = + wasm_instance_new(store, deserialized, &imports, NULL); + if (!instance) { + printf("> Error instantiating module!\n"); + return 1; + } + + wasm_func_delete(hello_func); + + // Extract export. + printf("Extracting export...\n"); + own wasm_extern_vec_t exports; + wasm_instance_exports(instance, &exports); + if (exports.size == 0) { + printf("> Error accessing exports!\n"); + return 1; + } + const wasm_func_t *run_func = wasm_extern_as_func(exports.data[0]); + if (run_func == NULL) { + printf("> Error accessing export!\n"); + return 1; + } + + wasm_module_delete(deserialized); + wasm_instance_delete(instance); + + // Call. + printf("Calling export...\n"); + wasm_val_vec_t empty = WASM_EMPTY_VEC; + if (wasm_func_call(run_func, &empty, &empty)) { + printf("> Error calling function!\n"); + return 1; + } + + wasm_extern_vec_delete(&exports); + + // Shut down. + printf("Shutting down...\n"); + wasm_store_delete(store); + wasm_engine_delete(engine); + + // All done. + printf("Done.\n"); + return 0; +} diff --git a/samples/wasm-c-api/src/serialize.wat b/samples/wasm-c-api/src/serialize.wat new file mode 100644 index 00000000..1c56c558 --- /dev/null +++ b/samples/wasm-c-api/src/serialize.wat @@ -0,0 +1,4 @@ +(module + (func $hello (import "" "hello")) + (func (export "run") (call $hello)) +)