From a182926a73483569444a6ef87eacf8981b0aefa5 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Tue, 18 Oct 2022 10:59:28 +0800 Subject: [PATCH] Refactor interpreter/AOT module instance layout (#1559) Refactor the layout of interpreter and AOT module instance: - Unify the interp/AOT module instance, use the same WASMModuleInstance/ WASMMemoryInstance/WASMTableInstance data structures for both interpreter and AOT - Make the offset of most fields the same in module instance for both interpreter and AOT, append memory instance structure, global data and table instances to the end of module instance for interpreter mode (like AOT mode) - For extra fields in WASM module instance, use WASMModuleInstanceExtra to create a field `e` for interpreter - Change the LLVM JIT module instance creating process, LLVM JIT uses the WASM module and module instance same as interpreter/Fast-JIT mode. So that Fast JIT and LLVM JIT can access the same data structures, and make it possible to implement the Multi-tier JIT (tier-up from Fast JIT to LLVM JIT) in the future - Unify some APIs: merge some APIs for module instance and memory instance's related operations (only implement one copy) Note that the AOT ABI is same, the AOT file format, AOT relocation types, how AOT code accesses the AOT module instance and so on are kept unchanged. Refer to: https://github.com/bytecodealliance/wasm-micro-runtime/issues/1384 --- .gitignore | 5 +- build-scripts/config_common.cmake | 57 +- build-scripts/runtime_lib.cmake | 30 +- core/iwasm/aot/aot_intrinsic.h | 2 +- core/iwasm/aot/aot_loader.c | 484 +----- core/iwasm/aot/aot_runtime.c | 975 +++-------- core/iwasm/aot/aot_runtime.h | 237 +-- core/iwasm/common/wasm_c_api.c | 186 ++- core/iwasm/common/wasm_exec_env.c | 4 +- core/iwasm/common/wasm_memory.c | 450 ++++++ core/iwasm/common/wasm_runtime_common.c | 727 ++++----- core/iwasm/common/wasm_runtime_common.h | 21 +- core/iwasm/common/wasm_shared_memory.c | 77 +- core/iwasm/compilation/aot.h | 12 +- core/iwasm/compilation/aot_compiler.c | 20 + core/iwasm/compilation/aot_emit_const.c | 2 +- core/iwasm/compilation/aot_emit_exception.c | 3 +- core/iwasm/compilation/aot_emit_function.c | 18 +- core/iwasm/compilation/aot_emit_memory.c | 30 +- core/iwasm/compilation/aot_emit_table.c | 57 +- core/iwasm/compilation/aot_emit_variable.c | 15 +- core/iwasm/compilation/aot_llvm.c | 31 +- core/iwasm/compilation/aot_llvm_extra.cpp | 4 +- core/iwasm/fast-jit/fe/jit_emit_control.c | 2 +- core/iwasm/fast-jit/fe/jit_emit_conversion.c | 12 +- core/iwasm/fast-jit/fe/jit_emit_exception.c | 2 +- core/iwasm/fast-jit/fe/jit_emit_function.c | 17 +- core/iwasm/fast-jit/fe/jit_emit_memory.c | 26 +- core/iwasm/fast-jit/fe/jit_emit_numberic.c | 12 +- core/iwasm/fast-jit/fe/jit_emit_table.c | 28 +- core/iwasm/fast-jit/fe/jit_emit_variable.c | 4 +- core/iwasm/fast-jit/jit_frontend.c | 41 +- core/iwasm/fast-jit/jit_frontend.h | 22 - core/iwasm/fast-jit/jit_ir.c | 7 +- core/iwasm/include/wasm_c_api.h | 2 +- core/iwasm/interpreter/wasm.h | 17 + core/iwasm/interpreter/wasm_interp_classic.c | 444 +++-- core/iwasm/interpreter/wasm_interp_fast.c | 184 +-- core/iwasm/interpreter/wasm_loader.c | 134 +- core/iwasm/interpreter/wasm_mini_loader.c | 159 +- core/iwasm/interpreter/wasm_runtime.c | 1422 +++++++++-------- core/iwasm/interpreter/wasm_runtime.h | 396 +++-- .../libraries/debug-engine/debug_engine.c | 10 +- product-mini/platforms/android/CMakeLists.txt | 3 +- samples/wasm-c-api/CMakeLists.txt | 7 + samples/wasm-c-api/src/clone.c | 518 ++++++ samples/wasm-c-api/src/clone.wat | 15 + samples/wasm-c-api/src/serialize.c | 129 ++ samples/wasm-c-api/src/serialize.wat | 4 + 49 files changed, 3790 insertions(+), 3274 deletions(-) create mode 100644 samples/wasm-c-api/src/clone.c create mode 100644 samples/wasm-c-api/src/clone.wat create mode 100644 samples/wasm-c-api/src/serialize.c create mode 100644 samples/wasm-c-api/src/serialize.wat diff --git a/.gitignore b/.gitignore index 8b9e2b7d..b0dc43c5 100644 --- a/.gitignore +++ b/.gitignore @@ -28,4 +28,7 @@ tests/wamr-test-suites/workspace samples/socket-api/wasm-src/inc/pthread.h -**/__pycache__ \ No newline at end of file +**/__pycache__ + +# ignore benchmarks generated +tests/benchmarks/coremark/coremark* diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index c99d1564..289242b2 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -3,7 +3,6 @@ string(TOUPPER ${WAMR_BUILD_TARGET} WAMR_BUILD_TARGET) - # Add definitions for the build target if (WAMR_BUILD_TARGET STREQUAL "X86_64") add_definitions(-DBUILD_TARGET_X86_64) @@ -85,33 +84,29 @@ endif () endif () if (WAMR_BUILD_JIT EQUAL 1) - if (WAMR_BUILD_AOT EQUAL 1) - add_definitions("-DWASM_ENABLE_JIT=1") - if (NOT WAMR_BUILD_LAZY_JIT EQUAL 0) - # Enable Lazy JIT by default - set (WAMR_BUILD_LAZY_JIT 1) - add_definitions("-DWASM_ENABLE_LAZY_JIT=1") + add_definitions("-DWASM_ENABLE_JIT=1") + if (NOT WAMR_BUILD_LAZY_JIT EQUAL 0) + # Enable Lazy JIT by default + set (WAMR_BUILD_LAZY_JIT 1) + add_definitions("-DWASM_ENABLE_LAZY_JIT=1") + endif () + if (NOT DEFINED LLVM_DIR) + set (LLVM_SRC_ROOT "${WAMR_ROOT_DIR}/core/deps/llvm") + set (LLVM_BUILD_ROOT "${LLVM_SRC_ROOT}/build") + if (WAMR_BUILD_PLATFORM STREQUAL "windows") + set (LLVM_BUILD_ROOT "${LLVM_SRC_ROOT}/win32build") endif () - if (NOT DEFINED LLVM_DIR) - set (LLVM_SRC_ROOT "${WAMR_ROOT_DIR}/core/deps/llvm") - set (LLVM_BUILD_ROOT "${LLVM_SRC_ROOT}/build") - if (WAMR_BUILD_PLATFORM STREQUAL "windows") - set (LLVM_BUILD_ROOT "${LLVM_SRC_ROOT}/win32build") - endif () - if (NOT EXISTS "${LLVM_BUILD_ROOT}") + if (NOT EXISTS "${LLVM_BUILD_ROOT}") message (FATAL_ERROR "Cannot find LLVM dir: ${LLVM_BUILD_ROOT}") - endif () - set (CMAKE_PREFIX_PATH "${LLVM_BUILD_ROOT};${CMAKE_PREFIX_PATH}") endif () - find_package(LLVM REQUIRED CONFIG) - include_directories(${LLVM_INCLUDE_DIRS}) - add_definitions(${LLVM_DEFINITIONS}) - message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") - message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") - else () - set (WAMR_BUILD_JIT 0) - message ("-- WAMR JIT disabled due to WAMR AOT is disabled") - endif () + set (CMAKE_PREFIX_PATH "${LLVM_BUILD_ROOT};${CMAKE_PREFIX_PATH}") + set (LLVM_DIR ${LLVM_BUILD_ROOT}/lib/cmake/llvm) + endif () + find_package(LLVM REQUIRED CONFIG) + include_directories(${LLVM_INCLUDE_DIRS}) + add_definitions(${LLVM_DEFINITIONS}) + message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") + message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") else () unset (LLVM_AVAILABLE_LIBS) endif () @@ -139,9 +134,6 @@ message (" Build as target ${WAMR_BUILD_TARGET}") message (" CMAKE_BUILD_TYPE " ${CMAKE_BUILD_TYPE}) if (WAMR_BUILD_INTERP EQUAL 1) message (" WAMR Interpreter enabled") -elseif (WAMR_BUILD_JIT EQUAL 1) - set (WAMR_BUILD_INTERP 1) - message (" WAMR Interpreter enabled due to WAMR JIT is enabled") else () message (" WAMR Interpreter disabled") endif () @@ -150,16 +142,19 @@ if (WAMR_BUILD_AOT EQUAL 1) else () message (" WAMR AOT disabled") endif () +if (WAMR_BUILD_FAST_JIT EQUAL 1) + message (" WAMR Fast JIT enabled") +else () + message (" WAMR Fast JIT disabled") +endif () if (WAMR_BUILD_JIT EQUAL 1) if (WAMR_BUILD_LAZY_JIT EQUAL 1) message (" WAMR LLVM Orc Lazy JIT enabled") else () message (" WAMR LLVM MC JIT enabled") endif () -elseif (WAMR_BUILD_FAST_JIT EQUAL 1) - message (" WAMR Fast JIT enabled") else () - message (" WAMR JIT disabled") + message (" WAMR LLVM JIT disabled") endif () if (WAMR_BUILD_LIBC_BUILTIN EQUAL 1) message (" Libc builtin enabled") diff --git a/build-scripts/runtime_lib.cmake b/build-scripts/runtime_lib.cmake index 2a46d432..c5195895 100644 --- a/build-scripts/runtime_lib.cmake +++ b/build-scripts/runtime_lib.cmake @@ -1,7 +1,6 @@ # Copyright (C) 2019 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - if (NOT DEFINED WAMR_ROOT_DIR) set (WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../) endif () @@ -50,23 +49,28 @@ if (NOT DEFINED WAMR_BUILD_TARGET) endif () ################ optional according to settings ################ -if (WAMR_BUILD_INTERP EQUAL 1 OR WAMR_BUILD_JIT EQUAL 1 - OR WAMR_BUILD_FAST_JIT EQUAL 1) - if (WAMR_BUILD_FAST_JIT EQUAL 1 OR WAMR_BUILD_JIT EQUAL 1) - set (WAMR_BUILD_FAST_INTERP 0) - endif () +if (WAMR_BUILD_FAST_JIT EQUAL 1 OR WAMR_BUILD_JIT EQUAL 1) + # Enable classic interpreter if Fast JIT or LLVM JIT is enabled + set (WAMR_BUILD_INTERP 1) + set (WAMR_BUILD_FAST_INTERP 0) +endif () + +if (WAMR_BUILD_INTERP EQUAL 1) include (${IWASM_DIR}/interpreter/iwasm_interp.cmake) endif () +if (WAMR_BUILD_FAST_JIT EQUAL 1) + include (${IWASM_DIR}/fast-jit/iwasm_fast_jit.cmake) +endif () + +if (WAMR_BUILD_JIT EQUAL 1) + # Enable AOT if LLVM JIT is enabled + set (WAMR_BUILD_AOT 1) + include (${IWASM_DIR}/compilation/iwasm_compl.cmake) +endif () + if (WAMR_BUILD_AOT EQUAL 1) include (${IWASM_DIR}/aot/iwasm_aot.cmake) - if (WAMR_BUILD_JIT EQUAL 1) - include (${IWASM_DIR}/compilation/iwasm_compl.cmake) - endif () -endif () - -if (NOT WAMR_BUILD_JIT EQUAL 1 AND WAMR_BUILD_FAST_JIT EQUAL 1) - include (${IWASM_DIR}/fast-jit/iwasm_fast_jit.cmake) endif () if (WAMR_BUILD_APP_FRAMEWORK EQUAL 1) diff --git a/core/iwasm/aot/aot_intrinsic.h b/core/iwasm/aot/aot_intrinsic.h index 92dccea0..c6765ee0 100644 --- a/core/iwasm/aot/aot_intrinsic.h +++ b/core/iwasm/aot/aot_intrinsic.h @@ -7,7 +7,7 @@ #define _AOT_INTRINSIC_H #include "aot_runtime.h" -#if WASM_ENABLE_WAMR_COMPILER != 0 +#if WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0 #include "aot_llvm.h" #endif diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index fca926ab..dca35685 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -10,10 +10,6 @@ #include "../common/wasm_runtime_common.h" #include "../common/wasm_native.h" #include "../compilation/aot.h" -#if WASM_ENABLE_JIT != 0 -#include "../compilation/aot_llvm.h" -#include "../interpreter/wasm_loader.h" -#endif #if WASM_ENABLE_DEBUG_AOT != 0 #include "debug/elf_parser.h" @@ -716,23 +712,19 @@ fail: } static void -destroy_import_memories(AOTImportMemory *import_memories, bool is_jit_mode) +destroy_import_memories(AOTImportMemory *import_memories) { - if (!is_jit_mode) - wasm_runtime_free(import_memories); + wasm_runtime_free(import_memories); } static void -destroy_mem_init_data_list(AOTMemInitData **data_list, uint32 count, - bool is_jit_mode) +destroy_mem_init_data_list(AOTMemInitData **data_list, uint32 count) { - if (!is_jit_mode) { - uint32 i; - for (i = 0; i < count; i++) - if (data_list[i]) - wasm_runtime_free(data_list[i]); - wasm_runtime_free(data_list); - } + uint32 i; + for (i = 0; i < count; i++) + if (data_list[i]) + wasm_runtime_free(data_list[i]); + wasm_runtime_free(data_list); } static bool @@ -828,30 +820,25 @@ fail: } static void -destroy_import_tables(AOTImportTable *import_tables, bool is_jit_mode) +destroy_import_tables(AOTImportTable *import_tables) { - if (!is_jit_mode) - wasm_runtime_free(import_tables); + wasm_runtime_free(import_tables); } static void -destroy_tables(AOTTable *tables, bool is_jit_mode) +destroy_tables(AOTTable *tables) { - if (!is_jit_mode) - wasm_runtime_free(tables); + wasm_runtime_free(tables); } static void -destroy_table_init_data_list(AOTTableInitData **data_list, uint32 count, - bool is_jit_mode) +destroy_table_init_data_list(AOTTableInitData **data_list, uint32 count) { - if (!is_jit_mode) { - uint32 i; - for (i = 0; i < count; i++) - if (data_list[i]) - wasm_runtime_free(data_list[i]); - wasm_runtime_free(data_list); - } + uint32 i; + for (i = 0; i < count; i++) + if (data_list[i]) + wasm_runtime_free(data_list[i]); + wasm_runtime_free(data_list); } static bool @@ -1003,15 +990,13 @@ fail: } static void -destroy_func_types(AOTFuncType **func_types, uint32 count, bool is_jit_mode) +destroy_func_types(AOTFuncType **func_types, uint32 count) { - if (!is_jit_mode) { - uint32 i; - for (i = 0; i < count; i++) - if (func_types[i]) - wasm_runtime_free(func_types[i]); - wasm_runtime_free(func_types); - } + uint32 i; + for (i = 0; i < count; i++) + if (func_types[i]) + wasm_runtime_free(func_types[i]); + wasm_runtime_free(func_types); } static bool @@ -1094,10 +1079,9 @@ fail: } static void -destroy_import_globals(AOTImportGlobal *import_globals, bool is_jit_mode) +destroy_import_globals(AOTImportGlobal *import_globals) { - if (!is_jit_mode) - wasm_runtime_free(import_globals); + wasm_runtime_free(import_globals); } static bool @@ -1177,10 +1161,9 @@ fail: } static void -destroy_globals(AOTGlobal *globals, bool is_jit_mode) +destroy_globals(AOTGlobal *globals) { - if (!is_jit_mode) - wasm_runtime_free(globals); + wasm_runtime_free(globals); } static bool @@ -1259,10 +1242,9 @@ fail: } static void -destroy_import_funcs(AOTImportFunc *import_funcs, bool is_jit_mode) +destroy_import_funcs(AOTImportFunc *import_funcs) { - if (!is_jit_mode) - wasm_runtime_free(import_funcs); + wasm_runtime_free(import_funcs); } static bool @@ -1652,10 +1634,9 @@ fail: } static void -destroy_exports(AOTExport *exports, bool is_jit_mode) +destroy_exports(AOTExport *exports) { - if (!is_jit_mode) - wasm_runtime_free(exports); + wasm_runtime_free(exports); } static bool @@ -2826,427 +2807,46 @@ aot_load_from_aot_file(const uint8 *buf, uint32 size, char *error_buf, return module; } -#if WASM_ENABLE_JIT != 0 -#if WASM_ENABLE_LAZY_JIT != 0 -/* Orc JIT thread arguments */ -typedef struct OrcJitThreadArg { - AOTCompData *comp_data; - AOTCompContext *comp_ctx; - AOTModule *module; - int32 group_idx; - int32 group_stride; -} OrcJitThreadArg; - -static bool orcjit_stop_compiling = false; -static korp_tid orcjit_threads[WASM_LAZY_JIT_COMPILE_THREAD_NUM]; -static OrcJitThreadArg orcjit_thread_args[WASM_LAZY_JIT_COMPILE_THREAD_NUM]; - -static void * -orcjit_thread_callback(void *arg) -{ - LLVMErrorRef error; - LLVMOrcJITTargetAddress func_addr = 0; - OrcJitThreadArg *thread_arg = (OrcJitThreadArg *)arg; - AOTCompData *comp_data = thread_arg->comp_data; - AOTCompContext *comp_ctx = thread_arg->comp_ctx; - AOTModule *module = thread_arg->module; - char func_name[32]; - int32 i; - - /* Compile wasm functions of this group */ - for (i = thread_arg->group_idx; i < (int32)comp_data->func_count; - i += thread_arg->group_stride) { - if (!module->func_ptrs[i]) { - snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i); - if ((error = LLVMOrcLLJITLookup(comp_ctx->orc_lazyjit, &func_addr, - func_name))) { - char *err_msg = LLVMGetErrorMessage(error); - os_printf("failed to compile orc jit function: %s", err_msg); - LLVMDisposeErrorMessage(err_msg); - break; - } - /** - * No need to lock the func_ptr[func_idx] here as it is basic - * data type, the load/store for it can be finished by one cpu - * instruction, and there can be only one cpu instruction - * loading/storing at the same time. - */ - module->func_ptrs[i] = (void *)func_addr; - } - if (orcjit_stop_compiling) { - break; - } - } - - /* Try to compile functions that haven't been compiled by other threads */ - for (i = (int32)comp_data->func_count - 1; i > 0; i--) { - if (orcjit_stop_compiling) { - break; - } - if (!module->func_ptrs[i]) { - snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i); - if ((error = LLVMOrcLLJITLookup(comp_ctx->orc_lazyjit, &func_addr, - func_name))) { - char *err_msg = LLVMGetErrorMessage(error); - os_printf("failed to compile orc jit function: %s", err_msg); - LLVMDisposeErrorMessage(err_msg); - break; - } - module->func_ptrs[i] = (void *)func_addr; - } - } - - return NULL; -} - -static void -orcjit_stop_compile_threads() -{ - uint32 i; - - orcjit_stop_compiling = true; - for (i = 0; i < WASM_LAZY_JIT_COMPILE_THREAD_NUM; i++) { - os_thread_join(orcjit_threads[i], NULL); - } -} -#endif - -static AOTModule * -aot_load_from_comp_data(AOTCompData *comp_data, AOTCompContext *comp_ctx, - char *error_buf, uint32 error_buf_size) -{ - uint32 i; - uint64 size; - char func_name[32]; - AOTModule *module; - - /* Allocate memory for module */ - if (!(module = - loader_malloc(sizeof(AOTModule), error_buf, error_buf_size))) { - return NULL; - } - - module->module_type = Wasm_Module_AoT; - - module->import_memory_count = comp_data->import_memory_count; - module->import_memories = comp_data->import_memories; - - module->memory_count = comp_data->memory_count; - if (module->memory_count) { - size = sizeof(AOTMemory) * (uint64)module->memory_count; - if (!(module->memories = - loader_malloc(size, error_buf, error_buf_size))) { - goto fail1; - } - - bh_memcpy_s(module->memories, (uint32)size, comp_data->memories, - (uint32)size); - } - - module->mem_init_data_list = comp_data->mem_init_data_list; - module->mem_init_data_count = comp_data->mem_init_data_count; - - module->import_table_count = comp_data->import_table_count; - module->import_tables = comp_data->import_tables; - - module->table_count = comp_data->table_count; - module->tables = comp_data->tables; - - module->table_init_data_list = comp_data->table_init_data_list; - module->table_init_data_count = comp_data->table_init_data_count; - - module->func_type_count = comp_data->func_type_count; - module->func_types = comp_data->func_types; - - module->import_global_count = comp_data->import_global_count; - module->import_globals = comp_data->import_globals; - - module->global_count = comp_data->global_count; - module->globals = comp_data->globals; - - module->global_count = comp_data->global_count; - module->globals = comp_data->globals; - - module->global_data_size = comp_data->global_data_size; - - module->import_func_count = comp_data->import_func_count; - module->import_funcs = comp_data->import_funcs; - - module->func_count = comp_data->func_count; - - /* Allocate memory for function pointers */ - size = (uint64)module->func_count * sizeof(void *); - if (size > 0 - && !(module->func_ptrs = - loader_malloc(size, error_buf, error_buf_size))) { - goto fail2; - } - -#if WASM_ENABLE_LAZY_JIT != 0 - /* Create threads to compile the wasm functions */ - for (i = 0; i < WASM_LAZY_JIT_COMPILE_THREAD_NUM; i++) { - orcjit_thread_args[i].comp_data = comp_data; - orcjit_thread_args[i].comp_ctx = comp_ctx; - orcjit_thread_args[i].module = module; - orcjit_thread_args[i].group_idx = (int32)i; - orcjit_thread_args[i].group_stride = WASM_LAZY_JIT_COMPILE_THREAD_NUM; - if (os_thread_create(&orcjit_threads[i], orcjit_thread_callback, - (void *)&orcjit_thread_args[i], - APP_THREAD_STACK_SIZE_DEFAULT) - != 0) { - uint32 j; - - set_error_buf(error_buf, error_buf_size, - "create orcjit compile thread failed"); - /* Terminate the threads created */ - orcjit_stop_compiling = true; - for (j = 0; j < i; j++) { - os_thread_join(orcjit_threads[j], NULL); - } - goto fail3; - } - } -#else - /* Resolve function addresses */ - bh_assert(comp_ctx->exec_engine); - for (i = 0; i < comp_data->func_count; i++) { - snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i); - if (!(module->func_ptrs[i] = (void *)LLVMGetFunctionAddress( - comp_ctx->exec_engine, func_name))) { - set_error_buf(error_buf, error_buf_size, - "get function address failed"); - goto fail3; - } - } -#endif /* WASM_ENABLE_LAZY_JIT != 0 */ - - /* Allocation memory for function type indexes */ - size = (uint64)module->func_count * sizeof(uint32); - if (size > 0 - && !(module->func_type_indexes = - loader_malloc(size, error_buf, error_buf_size))) { - goto fail4; - } - for (i = 0; i < comp_data->func_count; i++) - module->func_type_indexes[i] = comp_data->funcs[i]->func_type_index; - - module->export_count = comp_data->wasm_module->export_count; - module->exports = comp_data->wasm_module->exports; - - module->start_func_index = comp_data->start_func_index; - if (comp_data->start_func_index != (uint32)-1) { - bh_assert(comp_data->start_func_index - < module->import_func_count + module->func_count); - /* TODO: fix issue that start func cannot be import func */ - if (comp_data->start_func_index >= module->import_func_count) { -#if WASM_ENABLE_LAZY_JIT != 0 - if (!module->func_ptrs[comp_data->start_func_index - - module->import_func_count]) { - LLVMErrorRef error; - LLVMOrcJITTargetAddress func_addr = 0; - - snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, - comp_data->start_func_index - - module->import_func_count); - if ((error = LLVMOrcLLJITLookup(comp_ctx->orc_lazyjit, - &func_addr, func_name))) { - char *err_msg = LLVMGetErrorMessage(error); - set_error_buf_v(error_buf, error_buf_size, - "failed to compile orc jit function: %s", - err_msg); - LLVMDisposeErrorMessage(err_msg); - goto fail5; - } - module->func_ptrs[comp_data->start_func_index - - module->import_func_count] = - (void *)func_addr; - } -#endif - module->start_function = - module->func_ptrs[comp_data->start_func_index - - module->import_func_count]; - } - } - - module->malloc_func_index = comp_data->malloc_func_index; - module->free_func_index = comp_data->free_func_index; - module->retain_func_index = comp_data->retain_func_index; - - module->aux_data_end_global_index = comp_data->aux_data_end_global_index; - module->aux_data_end = comp_data->aux_data_end; - module->aux_heap_base_global_index = comp_data->aux_heap_base_global_index; - module->aux_heap_base = comp_data->aux_heap_base; - module->aux_stack_top_global_index = comp_data->aux_stack_top_global_index; - module->aux_stack_bottom = comp_data->aux_stack_bottom; - module->aux_stack_size = comp_data->aux_stack_size; - - module->code = NULL; - module->code_size = 0; - - module->is_jit_mode = true; - - module->wasm_module = comp_data->wasm_module; - module->comp_ctx = comp_ctx; - module->comp_data = comp_data; - -#if WASM_ENABLE_LIBC_WASI != 0 - module->import_wasi_api = comp_data->wasm_module->import_wasi_api; -#endif - - return module; - -#if WASM_ENABLE_LAZY_JIT != 0 -fail5: - if (module->func_type_indexes) - wasm_runtime_free(module->func_type_indexes); -#endif - -fail4: -#if WASM_ENABLE_LAZY_JIT != 0 - /* Terminate all threads before free module->func_ptrs */ - orcjit_stop_compile_threads(); -#endif -fail3: - if (module->func_ptrs) - wasm_runtime_free(module->func_ptrs); -fail2: - if (module->memory_count > 0) - wasm_runtime_free(module->memories); -fail1: - wasm_runtime_free(module); - return NULL; -} - -AOTModule * -aot_convert_wasm_module(WASMModule *wasm_module, char *error_buf, - uint32 error_buf_size) -{ - AOTCompData *comp_data; - AOTCompContext *comp_ctx; - AOTModule *aot_module; - AOTCompOption option = { 0 }; - char *aot_last_error; - - comp_data = aot_create_comp_data(wasm_module); - if (!comp_data) { - aot_last_error = aot_get_last_error(); - bh_assert(aot_last_error != NULL); - set_error_buf(error_buf, error_buf_size, aot_last_error); - return NULL; - } - - option.is_jit_mode = true; - option.opt_level = 3; - option.size_level = 3; -#if WASM_ENABLE_BULK_MEMORY != 0 - option.enable_bulk_memory = true; -#endif -#if WASM_ENABLE_THREAD_MGR != 0 - option.enable_thread_mgr = true; -#endif -#if WASM_ENABLE_TAIL_CALL != 0 - option.enable_tail_call = true; -#endif -#if WASM_ENABLE_SIMD != 0 - option.enable_simd = true; -#endif -#if WASM_ENABLE_REF_TYPES != 0 - option.enable_ref_types = true; -#endif - option.enable_aux_stack_check = true; -#if (WASM_ENABLE_PERF_PROFILING != 0) || (WASM_ENABLE_DUMP_CALL_STACK != 0) - option.enable_aux_stack_frame = true; -#endif - - comp_ctx = aot_create_comp_context(comp_data, &option); - if (!comp_ctx) { - aot_last_error = aot_get_last_error(); - bh_assert(aot_last_error != NULL); - set_error_buf(error_buf, error_buf_size, aot_last_error); - goto fail1; - } - - if (!aot_compile_wasm(comp_ctx)) { - aot_last_error = aot_get_last_error(); - bh_assert(aot_last_error != NULL); - set_error_buf(error_buf, error_buf_size, aot_last_error); - goto fail2; - } - - aot_module = - aot_load_from_comp_data(comp_data, comp_ctx, error_buf, error_buf_size); - if (!aot_module) { - goto fail2; - } - - return aot_module; - -fail2: - aot_destroy_comp_context(comp_ctx); -fail1: - aot_destroy_comp_data(comp_data); - return NULL; -} -#endif - void aot_unload(AOTModule *module) { -#if WASM_ENABLE_JIT != 0 -#if WASM_ENABLE_LAZY_JIT != 0 - orcjit_stop_compile_threads(); -#endif - - if (module->comp_data) - aot_destroy_comp_data(module->comp_data); - - if (module->comp_ctx) - aot_destroy_comp_context(module->comp_ctx); - - if (module->wasm_module) - wasm_loader_unload(module->wasm_module); -#endif - if (module->import_memories) - destroy_import_memories(module->import_memories, module->is_jit_mode); + destroy_import_memories(module->import_memories); if (module->memories) wasm_runtime_free(module->memories); if (module->mem_init_data_list) destroy_mem_init_data_list(module->mem_init_data_list, - module->mem_init_data_count, - module->is_jit_mode); + module->mem_init_data_count); if (module->native_symbol_list) wasm_runtime_free(module->native_symbol_list); if (module->import_tables) - destroy_import_tables(module->import_tables, module->is_jit_mode); + destroy_import_tables(module->import_tables); if (module->tables) - destroy_tables(module->tables, module->is_jit_mode); + destroy_tables(module->tables); if (module->table_init_data_list) destroy_table_init_data_list(module->table_init_data_list, - module->table_init_data_count, - module->is_jit_mode); + module->table_init_data_count); if (module->func_types) - destroy_func_types(module->func_types, module->func_type_count, - module->is_jit_mode); + destroy_func_types(module->func_types, module->func_type_count); if (module->import_globals) - destroy_import_globals(module->import_globals, module->is_jit_mode); + destroy_import_globals(module->import_globals); if (module->globals) - destroy_globals(module->globals, module->is_jit_mode); + destroy_globals(module->globals); if (module->import_funcs) - destroy_import_funcs(module->import_funcs, module->is_jit_mode); + destroy_import_funcs(module->import_funcs); if (module->exports) - destroy_exports(module->exports, module->is_jit_mode); + destroy_exports(module->exports); if (module->func_type_indexes) wasm_runtime_free(module->func_type_indexes); diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index c1ae5e41..13bdc920 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -7,6 +7,7 @@ #include "bh_log.h" #include "mem_alloc.h" #include "../common/wasm_runtime_common.h" +#include "../interpreter/wasm_runtime.h" #if WASM_ENABLE_SHARED_MEMORY != 0 #include "../common/wasm_shared_memory.h" #endif @@ -30,6 +31,15 @@ bh_static_assert(offsetof(WASMExecEnv, aux_stack_bottom) == 7 * sizeof(uintptr_t)); bh_static_assert(offsetof(WASMExecEnv, native_symbol) == 8 * sizeof(uintptr_t)); +bh_static_assert(offsetof(AOTModuleInstance, memories) == 1 * sizeof(uint64)); +bh_static_assert(offsetof(AOTModuleInstance, func_ptrs) == 5 * sizeof(uint64)); +bh_static_assert(offsetof(AOTModuleInstance, func_type_indexes) + == 6 * sizeof(uint64)); +bh_static_assert(offsetof(AOTModuleInstance, cur_exception) + == 13 * sizeof(uint64)); +bh_static_assert(offsetof(AOTModuleInstance, global_table_data) + == 13 * sizeof(uint64) + 128 + 11 * sizeof(uint64)); + static void set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) { @@ -114,7 +124,7 @@ init_global_data(uint8 *global_data, uint8 type, WASMValue *initial_value) break; #if WASM_ENABLE_SIMD != 0 case VALUE_TYPE_V128: - bh_memcpy_s(global_data, sizeof(V128), &initial_value->i64, + bh_memcpy_s(global_data, sizeof(V128), &initial_value->v128, sizeof(V128)); break; #endif @@ -129,14 +139,14 @@ global_instantiate(AOTModuleInstance *module_inst, AOTModule *module, { uint32 i; InitializerExpression *init_expr; - uint8 *p = (uint8 *)module_inst->global_data.ptr; + uint8 *p = module_inst->global_data; AOTImportGlobal *import_global = module->import_globals; AOTGlobal *global = module->globals; /* Initialize import global data */ for (i = 0; i < module->import_global_count; i++, import_global++) { bh_assert(import_global->data_offset - == (uint32)(p - (uint8 *)module_inst->global_data.ptr)); + == (uint32)(p - module_inst->global_data)); init_global_data(p, import_global->type, &import_global->global_data_linked); p += import_global->size; @@ -145,7 +155,7 @@ global_instantiate(AOTModuleInstance *module_inst, AOTModule *module, /* Initialize defined global data */ for (i = 0; i < module->global_count; i++, global++) { bh_assert(global->data_offset - == (uint32)(p - (uint8 *)module_inst->global_data.ptr)); + == (uint32)(p - module_inst->global_data)); init_expr = &global->init_expr; switch (init_expr->init_expr_type) { case INIT_EXPR_TYPE_GET_GLOBAL: @@ -177,39 +187,26 @@ global_instantiate(AOTModuleInstance *module_inst, AOTModule *module, } bh_assert(module_inst->global_data_size - == (uint32)(p - (uint8 *)module_inst->global_data.ptr)); + == (uint32)(p - module_inst->global_data)); return true; } -AOTTableInstance * -aot_next_tbl_inst(const AOTTableInstance *tbl_inst) -{ - uint32 offset = offsetof(AOTTableInstance, data); - offset += tbl_inst->max_size * sizeof(uint32); - return (AOTTableInstance *)((uint8 *)tbl_inst + offset); -} - -static inline AOTTableInstance * -aot_get_table_inst(const AOTModuleInstance *module_inst, uint32 tbl_idx) -{ - uint32 i = 0; - AOTTableInstance *tbl_inst = (AOTTableInstance *)module_inst->tables.ptr; - - while (i != tbl_idx) { - tbl_inst = aot_next_tbl_inst(tbl_inst); - ++i; - } - - return tbl_inst; -} - static bool -table_instantiate(AOTModuleInstance *module_inst, AOTModule *module, - char *error_buf, uint32 error_buf_size) +tables_instantiate(AOTModuleInstance *module_inst, AOTModule *module, + AOTTableInstance *first_tbl_inst, char *error_buf, + uint32 error_buf_size) { uint32 i, global_index, global_data_offset, base_offset, length; + uint64 total_size; AOTTableInitData *table_seg; - AOTTableInstance *tbl_inst = (AOTTableInstance *)module_inst->tables.ptr; + AOTTableInstance *tbl_inst = first_tbl_inst; + + total_size = (uint64)sizeof(WASMTableInstance *) * module_inst->table_count; + if (total_size > 0 + && !(module_inst->tables = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return false; + } /* * treat import table like a local one until we enable module linking @@ -219,15 +216,22 @@ table_instantiate(AOTModuleInstance *module_inst, AOTModule *module, if (i < module->import_table_count) { AOTImportTable *import_table = module->import_tables + i; tbl_inst->cur_size = import_table->table_init_size; - tbl_inst->max_size = aot_get_imp_tbl_data_slots(import_table); + tbl_inst->max_size = + aot_get_imp_tbl_data_slots(import_table, false); } else { AOTTable *table = module->tables + (i - module->import_table_count); tbl_inst->cur_size = table->table_init_size; - tbl_inst->max_size = aot_get_tbl_data_slots(table); + tbl_inst->max_size = aot_get_tbl_data_slots(table, false); } - tbl_inst = aot_next_tbl_inst(tbl_inst); + /* Set all elements to -1 to mark them as uninitialized elements */ + memset(tbl_inst->elems, 0xff, sizeof(uint32) * tbl_inst->max_size); + + module_inst->tables[i] = tbl_inst; + tbl_inst = (AOTTableInstance *)((uint8 *)tbl_inst + + offsetof(AOTTableInstance, elems) + + sizeof(uint32) * tbl_inst->max_size); } /* fill table with element segment content */ @@ -241,7 +245,7 @@ table_instantiate(AOTModuleInstance *module_inst, AOTModule *module, bh_assert(table_seg->table_index < module_inst->table_count); - tbl_inst = aot_get_table_inst(module_inst, table_seg->table_index); + tbl_inst = module_inst->tables[table_seg->table_index]; bh_assert(tbl_inst); #if WASM_ENABLE_REF_TYPES != 0 @@ -274,8 +278,8 @@ table_instantiate(AOTModuleInstance *module_inst, AOTModule *module, module->globals[global_index - module->import_global_count] .data_offset; - base_offset = *(uint32 *)((uint8 *)module_inst->global_data.ptr - + global_data_offset); + base_offset = + *(uint32 *)(module_inst->global_data + global_data_offset); } else base_offset = (uint32)table_seg->offset.u.i32; @@ -310,7 +314,7 @@ table_instantiate(AOTModuleInstance *module_inst, AOTModule *module, * Check function index in the current module inst for now. * will check the linked table inst owner in future */ - bh_memcpy_s((uint32 *)tbl_inst->data + base_offset, + bh_memcpy_s(tbl_inst->elems + base_offset, (tbl_inst->max_size - base_offset) * sizeof(uint32), table_seg->func_indexes, length * sizeof(uint32)); } @@ -325,12 +329,12 @@ memories_deinstantiate(AOTModuleInstance *module_inst) AOTMemoryInstance *memory_inst; for (i = 0; i < module_inst->memory_count; i++) { - memory_inst = ((AOTMemoryInstance **)module_inst->memories.ptr)[i]; + memory_inst = module_inst->memories[i]; if (memory_inst) { #if WASM_ENABLE_SHARED_MEMORY != 0 if (memory_inst->is_shared) { int32 ref_count = shared_memory_dec_reference( - (WASMModuleCommon *)module_inst->aot_module.ptr); + (WASMModuleCommon *)module_inst->module); bh_assert(ref_count >= 0); /* if the reference count is not zero, @@ -339,27 +343,26 @@ memories_deinstantiate(AOTModuleInstance *module_inst) continue; } #endif - if (memory_inst->heap_handle.ptr) { - mem_allocator_destroy(memory_inst->heap_handle.ptr); - wasm_runtime_free(memory_inst->heap_handle.ptr); + if (memory_inst->heap_handle) { + mem_allocator_destroy(memory_inst->heap_handle); + wasm_runtime_free(memory_inst->heap_handle); } - if (memory_inst->memory_data.ptr) { + if (memory_inst->memory_data) { #ifndef OS_ENABLE_HW_BOUND_CHECK - wasm_runtime_free(memory_inst->memory_data.ptr); + wasm_runtime_free(memory_inst->memory_data); #else #ifdef BH_PLATFORM_WINDOWS - os_mem_decommit(memory_inst->memory_data.ptr, + os_mem_decommit(memory_inst->memory_data, memory_inst->num_bytes_per_page * memory_inst->cur_page_count); #endif - os_munmap((uint8 *)memory_inst->memory_data.ptr, - 8 * (uint64)BH_GB); + os_munmap(memory_inst->memory_data, 8 * (uint64)BH_GB); #endif } } } - wasm_runtime_free(module_inst->memories.ptr); + wasm_runtime_free(module_inst->memories); } static AOTMemoryInstance * @@ -461,7 +464,7 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, /* Adjust __heap_base global value */ global_idx = module->aux_heap_base_global_index - module->import_global_count; - global_addr = (uint8 *)module_inst->global_data.ptr + global_addr = module_inst->global_data + module->globals[global_idx].data_offset; *(uint32 *)global_addr = aux_heap_base; LOG_VERBOSE("Reset __heap_base global to %u", aux_heap_base); @@ -557,12 +560,12 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, memory_inst->memory_data_size = (uint32)total_size; /* Init memory info */ - memory_inst->memory_data.ptr = p; - memory_inst->memory_data_end.ptr = p + (uint32)total_size; + memory_inst->memory_data = p; + memory_inst->memory_data_end = p + (uint32)total_size; /* Initialize heap info */ - memory_inst->heap_data.ptr = p + heap_offset; - memory_inst->heap_data_end.ptr = p + heap_offset + heap_size; + memory_inst->heap_data = p + heap_offset; + memory_inst->heap_data_end = p + heap_offset + heap_size; if (heap_size > 0) { uint32 heap_struct_size = mem_allocator_get_heap_struct_size(); @@ -571,10 +574,10 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, goto fail1; } - memory_inst->heap_handle.ptr = heap_handle; + memory_inst->heap_handle = heap_handle; if (!mem_allocator_create_with_struct_and_pool( - heap_handle, heap_struct_size, memory_inst->heap_data.ptr, + heap_handle, heap_struct_size, memory_inst->heap_data, heap_size)) { set_error_buf(error_buf, error_buf_size, "init app heap failed"); goto fail2; @@ -614,31 +617,31 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, #if WASM_ENABLE_SHARED_MEMORY != 0 fail3: if (heap_size > 0) - mem_allocator_destroy(memory_inst->heap_handle.ptr); + mem_allocator_destroy(memory_inst->heap_handle); #endif fail2: if (heap_size > 0) - wasm_runtime_free(memory_inst->heap_handle.ptr); + wasm_runtime_free(memory_inst->heap_handle); fail1: #ifndef OS_ENABLE_HW_BOUND_CHECK - if (memory_inst->memory_data.ptr) - wasm_runtime_free(memory_inst->memory_data.ptr); + if (memory_inst->memory_data) + wasm_runtime_free(memory_inst->memory_data); #else #ifdef BH_PLATFORM_WINDOWS - if (memory_inst->memory_data.ptr) + if (memory_inst->memory_data) os_mem_decommit(p, total_size); #endif os_munmap(mapped_mem, map_size); #endif - memory_inst->memory_data.ptr = NULL; + memory_inst->memory_data = NULL; return NULL; } static AOTMemoryInstance * aot_get_default_memory(AOTModuleInstance *module_inst) { - if (module_inst->memories.ptr) - return ((AOTMemoryInstance **)module_inst->memories.ptr)[0]; + if (module_inst->memories) + return module_inst->memories[0]; else return NULL; } @@ -654,8 +657,8 @@ memories_instantiate(AOTModuleInstance *module_inst, AOTModule *module, uint64 total_size; module_inst->memory_count = memory_count; - total_size = sizeof(AOTPointer) * (uint64)memory_count; - if (!(module_inst->memories.ptr = + total_size = sizeof(AOTMemoryInstance *) * (uint64)memory_count; + if (!(module_inst->memories = runtime_malloc(total_size, error_buf, error_buf_size))) { return false; } @@ -669,7 +672,7 @@ memories_instantiate(AOTModuleInstance *module_inst, AOTModule *module, return false; } - ((AOTMemoryInstance **)module_inst->memories.ptr)[i] = memory_inst; + module_inst->memories[i] = memory_inst; } /* Get default memory instance */ @@ -707,15 +710,15 @@ memories_instantiate(AOTModuleInstance *module_inst, AOTModule *module, module->globals[global_index - module->import_global_count] .data_offset; - base_offset = *(uint32 *)((uint8 *)module_inst->global_data.ptr - + global_data_offset); + base_offset = + *(uint32 *)(module_inst->global_data + global_data_offset); } else { base_offset = (uint32)data_seg->offset.u.i32; } /* Copy memory data */ - bh_assert(memory_inst->memory_data.ptr + bh_assert(memory_inst->memory_data || memory_inst->memory_data_size == 0); /* Check memory data */ @@ -748,8 +751,8 @@ memories_instantiate(AOTModuleInstance *module_inst, AOTModule *module, return false; } - if (memory_inst->memory_data.ptr) { - bh_memcpy_s((uint8 *)memory_inst->memory_data.ptr + base_offset, + if (memory_inst->memory_data) { + bh_memcpy_s((uint8 *)memory_inst->memory_data + base_offset, memory_inst->memory_data_size - base_offset, data_seg->bytes, length); } @@ -771,13 +774,13 @@ init_func_ptrs(AOTModuleInstance *module_inst, AOTModule *module, return true; /* Allocate memory */ - if (!(module_inst->func_ptrs.ptr = + if (!(module_inst->func_ptrs = runtime_malloc(total_size, error_buf, error_buf_size))) { return false; } /* Set import function pointers */ - func_ptrs = (void **)module_inst->func_ptrs.ptr; + func_ptrs = (void **)module_inst->func_ptrs; for (i = 0; i < module->import_func_count; i++, func_ptrs++) { *func_ptrs = (void *)module->import_funcs[i].func_ptr_linked; if (!*func_ptrs) { @@ -807,13 +810,13 @@ init_func_type_indexes(AOTModuleInstance *module_inst, AOTModule *module, return true; /* Allocate memory */ - if (!(module_inst->func_type_indexes.ptr = + if (!(module_inst->func_type_indexes = runtime_malloc(total_size, error_buf, error_buf_size))) { return false; } /* Set import function type indexes */ - func_type_index = (uint32 *)module_inst->func_type_indexes.ptr; + func_type_index = module_inst->func_type_indexes; for (i = 0; i < module->import_func_count; i++, func_type_index++) *func_type_index = module->import_funcs[i].func_type_index; @@ -835,10 +838,10 @@ create_export_funcs(AOTModuleInstance *module_inst, AOTModule *module, /* Allocate memory */ size = sizeof(AOTFunctionInstance) * (uint64)module_inst->export_func_count; - if (!(module_inst->export_funcs.ptr = export_func = - runtime_malloc(size, error_buf, error_buf_size))) { + if (!(export_func = runtime_malloc(size, error_buf, error_buf_size))) { return false; } + module_inst->export_functions = (void *)export_func; for (i = 0; i < module->export_count; i++) { if (exports[i].kind == EXPORT_KIND_FUNC) { @@ -883,10 +886,10 @@ create_exports(AOTModuleInstance *module_inst, AOTModule *module, module_inst->export_global_count++; break; case EXPORT_KIND_TABLE: - module_inst->export_tab_count++; + module_inst->export_table_count++; break; case EXPORT_KIND_MEMORY: - module_inst->export_mem_count++; + module_inst->export_memory_count++; break; default: return false; @@ -931,7 +934,7 @@ execute_post_inst_function(AOTModuleInstance *module_inst) static bool execute_start_function(AOTModuleInstance *module_inst) { - AOTModule *module = (AOTModule *)module_inst->aot_module.ptr; + AOTModule *module = (AOTModule *)module_inst->module; WASMExecEnv *exec_env; typedef void (*F)(WASMExecEnv *); union { @@ -998,16 +1001,17 @@ aot_instantiate(AOTModule *module, bool is_sub_inst, uint32 stack_size, * calculate size of table data */ for (i = 0; i != module->import_table_count; ++i) { - table_size += offsetof(AOTTableInstance, data); - table_size += - (uint64)sizeof(uint32) - * (uint64)aot_get_imp_tbl_data_slots(module->import_tables + i); + table_size += offsetof(AOTTableInstance, elems); + table_size += (uint64)sizeof(uint32) + * (uint64)aot_get_imp_tbl_data_slots( + module->import_tables + i, false); } for (i = 0; i != module->table_count; ++i) { - table_size += offsetof(AOTTableInstance, data); - table_size += (uint64)sizeof(uint32) - * (uint64)aot_get_tbl_data_slots(module->tables + i); + table_size += offsetof(AOTTableInstance, elems); + table_size += + (uint64)sizeof(uint32) + * (uint64)aot_get_tbl_data_slots(module->tables + i, false); } total_size += table_size; @@ -1018,23 +1022,21 @@ aot_instantiate(AOTModule *module, bool is_sub_inst, uint32 stack_size, } module_inst->module_type = Wasm_Module_AoT; - module_inst->aot_module.ptr = module; + module_inst->module = (void *)module; /* Initialize global info */ p = (uint8 *)module_inst + module_inst_struct_size + module_inst_mem_inst_size; - module_inst->global_data.ptr = p; + module_inst->global_data = p; module_inst->global_data_size = module->global_data_size; if (!global_instantiate(module_inst, module, error_buf, error_buf_size)) goto fail; /* Initialize table info */ p += module->global_data_size; - module_inst->tables.ptr = p; module_inst->table_count = module->table_count + module->import_table_count; - /* Set all elements to -1 to mark them as uninitialized elements */ - memset(module_inst->tables.ptr, 0xff, (uint32)table_size); - if (!table_instantiate(module_inst, module, error_buf, error_buf_size)) + if (!tables_instantiate(module_inst, module, (AOTTableInstance *)p, + error_buf, error_buf_size)) goto fail; /* Initialize memory space */ @@ -1082,14 +1084,14 @@ aot_instantiate(AOTModule *module, bool is_sub_inst, uint32 stack_size, #if WASM_ENABLE_PERF_PROFILING != 0 total_size = (uint64)sizeof(AOTFuncPerfProfInfo) * (module->import_func_count + module->func_count); - if (!(module_inst->func_perf_profilings.ptr = + if (!(module_inst->func_perf_profilings = runtime_malloc(total_size, error_buf, error_buf_size))) { goto fail; } #endif #if WASM_ENABLE_DUMP_CALL_STACK != 0 - if (!(module_inst->frames.ptr = + if (!(module_inst->frames = runtime_malloc(sizeof(Vector), error_buf, error_buf_size))) { goto fail; } @@ -1133,18 +1135,6 @@ fail: return NULL; } -bool -aot_create_exec_env_singleton(AOTModuleInstance *module_inst) -{ - WASMExecEnv *exec_env = - wasm_exec_env_create((WASMModuleInstanceCommon *)module_inst, - module_inst->default_wasm_stack_size); - if (exec_env) - module_inst->exec_env_singleton.ptr = exec_env; - - return exec_env ? true : false; -} - void aot_deinstantiate(AOTModuleInstance *module_inst, bool is_sub_inst) { @@ -1159,33 +1149,35 @@ aot_deinstantiate(AOTModuleInstance *module_inst, bool is_sub_inst) #endif #if WASM_ENABLE_PERF_PROFILING != 0 - if (module_inst->func_perf_profilings.ptr) - wasm_runtime_free(module_inst->func_perf_profilings.ptr); + if (module_inst->func_perf_profilings) + wasm_runtime_free(module_inst->func_perf_profilings); #endif #if WASM_ENABLE_DUMP_CALL_STACK != 0 - if (module_inst->frames.ptr) { - bh_vector_destroy(module_inst->frames.ptr); - wasm_runtime_free(module_inst->frames.ptr); - module_inst->frames.ptr = NULL; + if (module_inst->frames) { + bh_vector_destroy(module_inst->frames); + wasm_runtime_free(module_inst->frames); + module_inst->frames = NULL; } #endif - if (module_inst->memories.ptr) + if (module_inst->tables) + wasm_runtime_free(module_inst->tables); + + if (module_inst->memories) memories_deinstantiate(module_inst); - if (module_inst->export_funcs.ptr) - wasm_runtime_free(module_inst->export_funcs.ptr); + if (module_inst->export_functions) + wasm_runtime_free(module_inst->export_functions); - if (module_inst->func_ptrs.ptr) - wasm_runtime_free(module_inst->func_ptrs.ptr); + if (module_inst->func_ptrs) + wasm_runtime_free(module_inst->func_ptrs); - if (module_inst->func_type_indexes.ptr) - wasm_runtime_free(module_inst->func_type_indexes.ptr); + if (module_inst->func_type_indexes) + wasm_runtime_free(module_inst->func_type_indexes); - if (module_inst->exec_env_singleton.ptr) - wasm_exec_env_destroy( - (WASMExecEnv *)module_inst->exec_env_singleton.ptr); + if (module_inst->exec_env_singleton) + wasm_exec_env_destroy((WASMExecEnv *)module_inst->exec_env_singleton); wasm_runtime_free(module_inst); } @@ -1196,7 +1188,7 @@ aot_lookup_function(const AOTModuleInstance *module_inst, const char *name, { uint32 i; AOTFunctionInstance *export_funcs = - (AOTFunctionInstance *)module_inst->export_funcs.ptr; + (AOTFunctionInstance *)module_inst->export_functions; for (i = 0; i < module_inst->export_func_count; i++) if (!strcmp(export_funcs[i].func_name, name)) @@ -1207,115 +1199,6 @@ aot_lookup_function(const AOTModuleInstance *module_inst, const char *name, #ifdef OS_ENABLE_HW_BOUND_CHECK -#ifndef BH_PLATFORM_WINDOWS -void -aot_signal_handler(WASMSignalInfo *sig_info) -{ - WASMExecEnv *exec_env_tls = sig_info->exec_env_tls; - void *sig_addr = sig_info->sig_addr; - AOTModuleInstance *module_inst; - AOTMemoryInstance *memory_inst; - WASMJmpBuf *jmpbuf_node; - uint8 *mapped_mem_start_addr = NULL; - uint8 *mapped_mem_end_addr = NULL; - uint8 *stack_min_addr; - uint32 page_size; - uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; - - /* Check whether current thread is running aot function */ - if (exec_env_tls && exec_env_tls->handle == os_self_thread() - && (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) { - /* Get mapped mem info of current instance */ - module_inst = (AOTModuleInstance *)exec_env_tls->module_inst; - /* Get the default memory instance */ - memory_inst = aot_get_default_memory(module_inst); - if (memory_inst) { - mapped_mem_start_addr = (uint8 *)memory_inst->memory_data.ptr; - mapped_mem_end_addr = - (uint8 *)memory_inst->memory_data.ptr + 8 * (uint64)BH_GB; - } - - /* Get stack info of current thread */ - page_size = os_getpagesize(); - stack_min_addr = os_thread_get_stack_boundary(); - - if (memory_inst - && (mapped_mem_start_addr <= (uint8 *)sig_addr - && (uint8 *)sig_addr < mapped_mem_end_addr)) { - /* The address which causes segmentation fault is inside - the memory instance's guard regions */ - aot_set_exception_with_id(module_inst, - EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS); - os_longjmp(jmpbuf_node->jmpbuf, 1); - } - else if (stack_min_addr - page_size <= (uint8 *)sig_addr - && (uint8 *)sig_addr - < stack_min_addr + page_size * guard_page_count) { - /* The address which causes segmentation fault is inside - native thread's guard page */ - aot_set_exception_with_id(module_inst, EXCE_NATIVE_STACK_OVERFLOW); - os_longjmp(jmpbuf_node->jmpbuf, 1); - } - } -} -#else /* else of BH_PLATFORM_WINDOWS */ -LONG -aot_exception_handler(WASMSignalInfo *sig_info) -{ - WASMExecEnv *exec_env_tls = sig_info->exec_env_tls; - EXCEPTION_POINTERS *exce_info = sig_info->exce_info; - PEXCEPTION_RECORD ExceptionRecord = exce_info->ExceptionRecord; - uint8 *sig_addr = (uint8 *)ExceptionRecord->ExceptionInformation[1]; - AOTModuleInstance *module_inst; - AOTMemoryInstance *memory_inst; - WASMJmpBuf *jmpbuf_node; - uint8 *mapped_mem_start_addr = NULL; - uint8 *mapped_mem_end_addr = NULL; - uint32 page_size = os_getpagesize(); - - if (exec_env_tls && exec_env_tls->handle == os_self_thread() - && (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) { - module_inst = (AOTModuleInstance *)exec_env_tls->module_inst; - if (ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { - /* Get the default memory instance */ - memory_inst = aot_get_default_memory(module_inst); - if (memory_inst) { - mapped_mem_start_addr = (uint8 *)memory_inst->memory_data.ptr; - mapped_mem_end_addr = - (uint8 *)memory_inst->memory_data.ptr + 8 * (uint64)BH_GB; - if (mapped_mem_start_addr <= (uint8 *)sig_addr - && (uint8 *)sig_addr < mapped_mem_end_addr) { - /* The address which causes segmentation fault is inside - aot instance's guard regions. - Set exception and let the aot func continue to run, when - the aot func returns, the caller will check whether the - exception is thrown and return to runtime. */ - aot_set_exception_with_id(module_inst, - EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS); - /* Skip current instruction */ - exce_info->ContextRecord->Rip++; - return EXCEPTION_CONTINUE_EXECUTION; - } - } - } - else if (ExceptionRecord->ExceptionCode == EXCEPTION_STACK_OVERFLOW) { - /* Set stack overflow exception and let the aot func continue - to run, when the aot func returns, the caller will check - whether the exception is thrown and return to runtime, and - the damaged stack will be recovered by _resetstkoflw(). */ - aot_set_exception_with_id(module_inst, EXCE_NATIVE_STACK_OVERFLOW); - return EXCEPTION_CONTINUE_EXECUTION; - } - } - - os_printf("Unhandled exception thrown: exception code: 0x%lx, " - "exception address: %p, exception information: %p\n", - ExceptionRecord->ExceptionCode, ExceptionRecord->ExceptionAddress, - sig_addr); - return EXCEPTION_CONTINUE_SEARCH; -} -#endif /* end of BH_PLATFORM_WINDOWS */ - static bool invoke_native_with_hw_bound_check(WASMExecEnv *exec_env, void *func_ptr, const WASMType *func_type, @@ -1438,17 +1321,6 @@ aot_call_function(WASMExecEnv *exec_env, AOTFunctionInstance *function, } argc = func_type->param_cell_num; -#if WASM_ENABLE_LAZY_JIT != 0 - if (!function->u.func.func_ptr) { - AOTModule *aot_module = (AOTModule *)module_inst->aot_module.ptr; - if (!(function->u.func.func_ptr = - aot_lookup_orcjit_func(aot_module->comp_ctx->orc_lazyjit, - module_inst, function->func_index))) { - return false; - } - } -#endif - /* set thread handle and stack boundary */ wasm_exec_env_set_thread_info(exec_env); @@ -1619,84 +1491,19 @@ aot_create_exec_env_and_call_function(AOTModuleInstance *module_inst, void aot_set_exception(AOTModuleInstance *module_inst, const char *exception) { - if (exception) - snprintf(module_inst->cur_exception, sizeof(module_inst->cur_exception), - "Exception: %s", exception); - else - module_inst->cur_exception[0] = '\0'; + wasm_set_exception(module_inst, exception); } void aot_set_exception_with_id(AOTModuleInstance *module_inst, uint32 id) { - switch (id) { - case EXCE_UNREACHABLE: - aot_set_exception(module_inst, "unreachable"); - break; - case EXCE_OUT_OF_MEMORY: - aot_set_exception(module_inst, "allocate memory failed"); - break; - case EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS: - aot_set_exception(module_inst, "out of bounds memory access"); - break; - case EXCE_INTEGER_OVERFLOW: - aot_set_exception(module_inst, "integer overflow"); - break; - case EXCE_INTEGER_DIVIDE_BY_ZERO: - aot_set_exception(module_inst, "integer divide by zero"); - break; - case EXCE_INVALID_CONVERSION_TO_INTEGER: - aot_set_exception(module_inst, "invalid conversion to integer"); - break; - case EXCE_INVALID_FUNCTION_TYPE_INDEX: - aot_set_exception(module_inst, "indirect call type mismatch"); - break; - case EXCE_INVALID_FUNCTION_INDEX: - aot_set_exception(module_inst, "invalid function index"); - break; - case EXCE_UNDEFINED_ELEMENT: - aot_set_exception(module_inst, "undefined element"); - break; - case EXCE_UNINITIALIZED_ELEMENT: - aot_set_exception(module_inst, "uninitialized element"); - break; - case EXCE_CALL_UNLINKED_IMPORT_FUNC: - aot_set_exception(module_inst, - "failed to call unlinked import function"); - break; - case EXCE_NATIVE_STACK_OVERFLOW: - aot_set_exception(module_inst, "native stack overflow"); - break; - case EXCE_UNALIGNED_ATOMIC: - aot_set_exception(module_inst, "unaligned atomic"); - break; - case EXCE_AUX_STACK_OVERFLOW: - aot_set_exception(module_inst, "wasm auxiliary stack overflow"); - break; - case EXCE_AUX_STACK_UNDERFLOW: - aot_set_exception(module_inst, "wasm auxiliary stack underflow"); - break; - case EXCE_OUT_OF_BOUNDS_TABLE_ACCESS: - aot_set_exception(module_inst, "out of bounds table access"); - break; - default: - break; - } + wasm_set_exception_with_id(module_inst, id); } const char * aot_get_exception(AOTModuleInstance *module_inst) { - if (module_inst->cur_exception[0] == '\0') - return NULL; - else - return module_inst->cur_exception; -} - -void -aot_clear_exception(AOTModuleInstance *module_inst) -{ - module_inst->cur_exception[0] = '\0'; + return wasm_get_exception(module_inst); } static bool @@ -1774,7 +1581,7 @@ aot_module_malloc(AOTModuleInstance *module_inst, uint32 size, void **p_native_addr) { AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - AOTModule *module = (AOTModule *)module_inst->aot_module.ptr; + AOTModule *module = (AOTModule *)module_inst->module; uint8 *addr = NULL; uint32 offset = 0; @@ -1783,8 +1590,8 @@ aot_module_malloc(AOTModuleInstance *module_inst, uint32 size, return 0; } - if (memory_inst->heap_handle.ptr) { - addr = mem_allocator_malloc(memory_inst->heap_handle.ptr, size); + if (memory_inst->heap_handle) { + addr = mem_allocator_malloc(memory_inst->heap_handle, size); } else if (module->malloc_func_index != (uint32)-1 && module->free_func_index != (uint32)-1) { @@ -1812,12 +1619,12 @@ aot_module_malloc(AOTModuleInstance *module_inst, uint32 size, size, &offset)) { return 0; } - addr = offset ? (uint8 *)memory_inst->memory_data.ptr + offset : NULL; + addr = offset ? (uint8 *)memory_inst->memory_data + offset : NULL; } if (!addr) { - if (memory_inst->heap_handle.ptr - && mem_allocator_is_heap_corrupted(memory_inst->heap_handle.ptr)) { + if (memory_inst->heap_handle + && mem_allocator_is_heap_corrupted(memory_inst->heap_handle)) { wasm_runtime_show_app_heap_corrupted_prompt(); aot_set_exception(module_inst, "app heap corrupted"); } @@ -1828,7 +1635,7 @@ aot_module_malloc(AOTModuleInstance *module_inst, uint32 size, } if (p_native_addr) *p_native_addr = addr; - return (uint32)(addr - (uint8 *)memory_inst->memory_data.ptr); + return (uint32)(addr - memory_inst->memory_data); } uint32 @@ -1843,17 +1650,17 @@ aot_module_realloc(AOTModuleInstance *module_inst, uint32 ptr, uint32 size, return 0; } - if (memory_inst->heap_handle.ptr) { + if (memory_inst->heap_handle) { addr = mem_allocator_realloc( - memory_inst->heap_handle.ptr, - ptr ? (uint8 *)memory_inst->memory_data.ptr + ptr : NULL, size); + memory_inst->heap_handle, + ptr ? memory_inst->memory_data + ptr : NULL, size); } /* Only support realloc in WAMR's app heap */ if (!addr) { - if (memory_inst->heap_handle.ptr - && mem_allocator_is_heap_corrupted(memory_inst->heap_handle.ptr)) { + if (memory_inst->heap_handle + && mem_allocator_is_heap_corrupted(memory_inst->heap_handle)) { aot_set_exception(module_inst, "app heap corrupted"); } else { @@ -1864,30 +1671,29 @@ aot_module_realloc(AOTModuleInstance *module_inst, uint32 ptr, uint32 size, if (p_native_addr) *p_native_addr = addr; - return (uint32)(addr - (uint8 *)memory_inst->memory_data.ptr); + return (uint32)(addr - memory_inst->memory_data); } void aot_module_free(AOTModuleInstance *module_inst, uint32 ptr) { AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - AOTModule *module = (AOTModule *)module_inst->aot_module.ptr; + AOTModule *module = (AOTModule *)module_inst->module; if (!memory_inst) { return; } if (ptr) { - uint8 *addr = (uint8 *)memory_inst->memory_data.ptr + ptr; - if (memory_inst->heap_handle.ptr - && (uint8 *)memory_inst->heap_data.ptr < addr - && addr < (uint8 *)memory_inst->heap_data_end.ptr) { - mem_allocator_free(memory_inst->heap_handle.ptr, addr); + uint8 *addr = memory_inst->memory_data + ptr; + if (memory_inst->heap_handle && memory_inst->heap_data < addr + && addr < memory_inst->heap_data_end) { + mem_allocator_free(memory_inst->heap_handle, addr); } else if (module->malloc_func_index != (uint32)-1 && module->free_func_index != (uint32)-1 - && (uint8 *)memory_inst->memory_data.ptr <= addr - && addr < (uint8 *)memory_inst->memory_data_end.ptr) { + && memory_inst->memory_data <= addr + && addr < memory_inst->memory_data_end) { AOTFunctionInstance *free_func; char *free_func_name; @@ -1917,331 +1723,18 @@ aot_module_dup_data(AOTModuleInstance *module_inst, const char *src, aot_module_malloc(module_inst, size, (void **)&buffer); if (buffer_offset != 0) { - buffer = aot_addr_app_to_native(module_inst, buffer_offset); + buffer = wasm_runtime_addr_app_to_native( + (WASMModuleInstanceCommon *)module_inst, buffer_offset); bh_memcpy_s(buffer, size, src, size); } return buffer_offset; } -bool -aot_validate_app_addr(AOTModuleInstance *module_inst, uint32 app_offset, - uint32 size) -{ - AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - - if (!memory_inst) { - goto fail; - } - - /* integer overflow check */ - if (app_offset > UINT32_MAX - size) { - goto fail; - } - - if (app_offset + size <= memory_inst->memory_data_size) { - return true; - } -fail: - aot_set_exception(module_inst, "out of bounds memory access"); - return false; -} - -bool -aot_validate_native_addr(AOTModuleInstance *module_inst, void *native_ptr, - uint32 size) -{ - AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - uint8 *addr = (uint8 *)native_ptr; - - if (!memory_inst) { - goto fail; - } - - /* integer overflow check */ - if ((uintptr_t)addr > UINTPTR_MAX - size) { - goto fail; - } - - if ((uint8 *)memory_inst->memory_data.ptr <= addr - && addr + size <= (uint8 *)memory_inst->memory_data_end.ptr) - return true; -fail: - aot_set_exception(module_inst, "out of bounds memory access"); - return false; -} - -void * -aot_addr_app_to_native(AOTModuleInstance *module_inst, uint32 app_offset) -{ - AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - uint8 *addr; - - if (!memory_inst) { - return NULL; - } - - addr = (uint8 *)memory_inst->memory_data.ptr + app_offset; - - if ((uint8 *)memory_inst->memory_data.ptr <= addr - && addr < (uint8 *)memory_inst->memory_data_end.ptr) - return addr; - return NULL; -} - -uint32 -aot_addr_native_to_app(AOTModuleInstance *module_inst, void *native_ptr) -{ - AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - uint8 *addr = (uint8 *)native_ptr; - - if (!memory_inst) { - return 0; - } - - if ((uint8 *)memory_inst->memory_data.ptr <= addr - && addr < (uint8 *)memory_inst->memory_data_end.ptr) - return (uint32)(addr - (uint8 *)memory_inst->memory_data.ptr); - return 0; -} - -bool -aot_get_app_addr_range(AOTModuleInstance *module_inst, uint32 app_offset, - uint32 *p_app_start_offset, uint32 *p_app_end_offset) -{ - AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - uint32 memory_data_size; - - if (!memory_inst) { - return false; - } - - memory_data_size = memory_inst->memory_data_size; - - if (app_offset < memory_data_size) { - if (p_app_start_offset) - *p_app_start_offset = 0; - if (p_app_end_offset) - *p_app_end_offset = memory_data_size; - return true; - } - return false; -} - -bool -aot_get_native_addr_range(AOTModuleInstance *module_inst, uint8 *native_ptr, - uint8 **p_native_start_addr, - uint8 **p_native_end_addr) -{ - AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - uint8 *addr = (uint8 *)native_ptr; - - if (!memory_inst) { - return false; - } - - if ((uint8 *)memory_inst->memory_data.ptr <= addr - && addr < (uint8 *)memory_inst->memory_data_end.ptr) { - if (p_native_start_addr) - *p_native_start_addr = (uint8 *)memory_inst->memory_data.ptr; - if (p_native_end_addr) - *p_native_end_addr = (uint8 *)memory_inst->memory_data_end.ptr; - return true; - } - return false; -} - -#ifndef OS_ENABLE_HW_BOUND_CHECK bool aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) { - AOTMemoryInstance *memory = aot_get_default_memory(module_inst); - uint8 *memory_data_old, *memory_data_new, *heap_data_old; - uint32 num_bytes_per_page, heap_size, total_size_old; - uint32 cur_page_count, max_page_count, total_page_count; - uint64 total_size_new; - bool ret = true; - - if (!memory) - return false; - - heap_data_old = (uint8 *)memory->heap_data.ptr; - heap_size = (uint32)((uint8 *)memory->heap_data_end.ptr - - (uint8 *)memory->heap_data.ptr); - - memory_data_old = (uint8 *)memory->memory_data.ptr; - total_size_old = - (uint32)((uint8 *)memory->memory_data_end.ptr - memory_data_old); - - num_bytes_per_page = memory->num_bytes_per_page; - cur_page_count = memory->cur_page_count; - max_page_count = memory->max_page_count; - total_page_count = inc_page_count + cur_page_count; - total_size_new = num_bytes_per_page * (uint64)total_page_count; - - if (inc_page_count <= 0) - /* No need to enlarge memory */ - return true; - - if (total_page_count < cur_page_count /* integer overflow */ - || total_page_count > max_page_count) { - return false; - } - - bh_assert(total_size_new <= 4 * (uint64)BH_GB); - if (total_size_new > UINT32_MAX) { - /* Resize to 1 page with size 4G-1 */ - num_bytes_per_page = UINT32_MAX; - total_page_count = max_page_count = 1; - total_size_new = UINT32_MAX; - } - -#if WASM_ENABLE_SHARED_MEMORY != 0 - if (memory->is_shared) { - memory->num_bytes_per_page = num_bytes_per_page; - memory->cur_page_count = total_page_count; - memory->max_page_count = max_page_count; - /* No need to update memory->memory_data_size as it is - initialized with the maximum memory data size for - shared memory */ - return true; - } -#endif - - if (heap_size > 0) { - if (mem_allocator_is_heap_corrupted(memory->heap_handle.ptr)) { - wasm_runtime_show_app_heap_corrupted_prompt(); - return false; - } - } - - if (!(memory_data_new = - wasm_runtime_realloc(memory_data_old, (uint32)total_size_new))) { - if (!(memory_data_new = wasm_runtime_malloc((uint32)total_size_new))) { - return false; - } - if (memory_data_old) { - bh_memcpy_s(memory_data_new, (uint32)total_size_new, - memory_data_old, total_size_old); - wasm_runtime_free(memory_data_old); - } - } - - memset(memory_data_new + total_size_old, 0, - (uint32)total_size_new - total_size_old); - - if (heap_size > 0) { - if (mem_allocator_migrate(memory->heap_handle.ptr, - (char *)heap_data_old - + (memory_data_new - memory_data_old), - heap_size) - != 0) { - /* Don't return here as memory->memory_data is obsolete and - must be updated to be correctly used later. */ - ret = false; - } - } - - memory->heap_data.ptr = memory_data_new + (heap_data_old - memory_data_old); - memory->heap_data_end.ptr = (uint8 *)memory->heap_data.ptr + heap_size; - - memory->num_bytes_per_page = num_bytes_per_page; - memory->cur_page_count = total_page_count; - memory->max_page_count = max_page_count; - - memory->memory_data.ptr = memory_data_new; - memory->memory_data_end.ptr = memory_data_new + (uint32)total_size_new; - memory->memory_data_size = (uint32)total_size_new; - -#if UINTPTR_MAX == UINT64_MAX - memory->mem_bound_check_1byte.u64 = total_size_new - 1; - memory->mem_bound_check_2bytes.u64 = total_size_new - 2; - memory->mem_bound_check_4bytes.u64 = total_size_new - 4; - memory->mem_bound_check_8bytes.u64 = total_size_new - 8; - memory->mem_bound_check_16bytes.u64 = total_size_new - 16; -#else - memory->mem_bound_check_1byte.u32[0] = (uint32)total_size_new - 1; - memory->mem_bound_check_2bytes.u32[0] = (uint32)total_size_new - 2; - memory->mem_bound_check_4bytes.u32[0] = (uint32)total_size_new - 4; - memory->mem_bound_check_8bytes.u32[0] = (uint32)total_size_new - 8; - memory->mem_bound_check_16bytes.u32[0] = (uint32)total_size_new - 16; -#endif - - return ret; + return wasm_enlarge_memory(module_inst, inc_page_count); } -#else /* else of OS_ENABLE_HW_BOUND_CHECK */ -bool -aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) -{ - AOTMemoryInstance *memory = aot_get_default_memory(module_inst); - uint32 num_bytes_per_page, total_size_old; - uint32 cur_page_count, max_page_count, total_page_count; - uint64 total_size_new; - - if (!memory) - return false; - - num_bytes_per_page = memory->num_bytes_per_page; - cur_page_count = memory->cur_page_count; - max_page_count = memory->max_page_count; - total_size_old = num_bytes_per_page * cur_page_count; - total_page_count = inc_page_count + cur_page_count; - total_size_new = num_bytes_per_page * (uint64)total_page_count; - - if (inc_page_count <= 0) - /* No need to enlarge memory */ - return true; - - if (total_page_count < cur_page_count /* integer overflow */ - || total_page_count > max_page_count) { - return false; - } - - bh_assert(total_size_new <= 4 * (uint64)BH_GB); - if (total_size_new > UINT32_MAX) { - /* Resize to 1 page with size 4G-1 */ - num_bytes_per_page = UINT32_MAX; - total_page_count = max_page_count = 1; - total_size_new = UINT32_MAX; - } - -#ifdef BH_PLATFORM_WINDOWS - if (!os_mem_commit(memory->memory_data_end.ptr, - (uint32)total_size_new - total_size_old, - MMAP_PROT_READ | MMAP_PROT_WRITE)) { - return false; - } -#endif - - if (os_mprotect(memory->memory_data_end.ptr, - (uint32)total_size_new - total_size_old, - MMAP_PROT_READ | MMAP_PROT_WRITE) - != 0) { -#ifdef BH_PLATFORM_WINDOWS - os_mem_decommit(memory->memory_data_end.ptr, - (uint32)total_size_new - total_size_old); -#endif - return false; - } - - /* The increased pages are filled with zero by the OS when os_mmap, - no need to memset it again here */ - - memory->num_bytes_per_page = num_bytes_per_page; - memory->cur_page_count = total_page_count; - memory->max_page_count = max_page_count; - memory->memory_data_size = (uint32)total_size_new; - memory->memory_data_end.ptr = - (uint8 *)memory->memory_data.ptr + (uint32)total_size_new; - - memory->mem_bound_check_1byte.u64 = total_size_new - 1; - memory->mem_bound_check_2bytes.u64 = total_size_new - 2; - memory->mem_bound_check_4bytes.u64 = total_size_new - 4; - memory->mem_bound_check_8bytes.u64 = total_size_new - 8; - memory->mem_bound_check_16bytes.u64 = total_size_new - 16; - - return true; -} -#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ bool aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc, @@ -2249,11 +1742,11 @@ aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc, { AOTModuleInstance *module_inst = (AOTModuleInstance *)wasm_runtime_get_module_inst(exec_env); - AOTModule *aot_module = (AOTModule *)module_inst->aot_module.ptr; - uint32 *func_type_indexes = (uint32 *)module_inst->func_type_indexes.ptr; + AOTModule *aot_module = (AOTModule *)module_inst->module; + uint32 *func_type_indexes = module_inst->func_type_indexes; uint32 func_type_idx = func_type_indexes[func_idx]; AOTFuncType *func_type = aot_module->func_types[func_type_idx]; - void **func_ptrs = (void **)module_inst->func_ptrs.ptr; + void **func_ptrs = module_inst->func_ptrs; void *func_ptr = func_ptrs[func_idx]; AOTImportFunc *import_func; const char *signature; @@ -2297,11 +1790,11 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, { AOTModuleInstance *module_inst = (AOTModuleInstance *)wasm_runtime_get_module_inst(exec_env); - AOTModule *aot_module = (AOTModule *)module_inst->aot_module.ptr; - uint32 *func_type_indexes = (uint32 *)module_inst->func_type_indexes.ptr; + AOTModule *aot_module = (AOTModule *)module_inst->module; + uint32 *func_type_indexes = module_inst->func_type_indexes; AOTTableInstance *tbl_inst; AOTFuncType *func_type; - void **func_ptrs = (void **)module_inst->func_ptrs.ptr, *func_ptr; + void **func_ptrs = module_inst->func_ptrs, *func_ptr; uint32 func_type_idx, func_idx, ext_ret_count; AOTImportFunc *import_func; const char *signature = NULL; @@ -2318,7 +1811,7 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, return false; } - tbl_inst = aot_get_table_inst(module_inst, tbl_idx); + tbl_inst = module_inst->tables[tbl_idx]; bh_assert(tbl_inst); if (table_elem_idx >= tbl_inst->cur_size) { @@ -2326,8 +1819,8 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, return false; } - func_idx = ((uint32 *)tbl_inst->data)[table_elem_idx]; - if (func_idx == (uint32)-1) { + func_idx = tbl_inst->elems[table_elem_idx]; + if (func_idx == NULL_REF) { aot_set_exception_with_id(module_inst, EXCE_UNINITIALIZED_ELEMENT); return false; } @@ -2335,15 +1828,6 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, func_type_idx = func_type_indexes[func_idx]; func_type = aot_module->func_types[func_type_idx]; -#if WASM_ENABLE_LAZY_JIT != 0 - if (func_idx >= aot_module->import_func_count && !func_ptrs[func_idx]) { - if (!(func_ptr = aot_lookup_orcjit_func( - aot_module->comp_ctx->orc_lazyjit, module_inst, func_idx))) { - return false; - } - } -#endif - if (!(func_ptr = func_ptrs[func_idx])) { bh_assert(func_idx < aot_module->import_func_count); import_func = aot_module->import_funcs + func_idx; @@ -2455,54 +1939,13 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, } } -/** - * Check whether the app address and the buf is inside the linear memory, - * and convert the app address into native address - */ bool aot_check_app_addr_and_convert(AOTModuleInstance *module_inst, bool is_str, uint32 app_buf_addr, uint32 app_buf_size, void **p_native_addr) { - AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - uint8 *native_addr; - - if (!memory_inst) { - goto fail; - } - - native_addr = (uint8 *)memory_inst->memory_data.ptr + app_buf_addr; - - /* No need to check the app_offset and buf_size if memory access - boundary check with hardware trap is enabled */ -#ifndef OS_ENABLE_HW_BOUND_CHECK - if (app_buf_addr >= memory_inst->memory_data_size) { - goto fail; - } - - if (!is_str) { - if (app_buf_size > memory_inst->memory_data_size - app_buf_addr) { - goto fail; - } - } - else { - const char *str, *str_end; - - /* The whole string must be in the linear memory */ - str = (const char *)native_addr; - str_end = (const char *)memory_inst->memory_data_end.ptr; - while (str < str_end && *str != '\0') - str++; - if (str == str_end) - goto fail; - } -#endif - - *p_native_addr = (void *)native_addr; - return true; -fail: - aot_set_exception(module_inst, "out of bounds memory access"); - return false; + return wasm_check_app_addr_and_convert(module_inst, is_str, app_buf_addr, + app_buf_size, p_native_addr); } void * @@ -2528,20 +1971,12 @@ aot_memory_init(AOTModuleInstance *module_inst, uint32 seg_index, uint32 offset, uint8 *maddr; uint64 seg_len = 0; - aot_module = (AOTModule *)module_inst->aot_module.ptr; - if (aot_module->is_jit_mode) { -#if WASM_ENABLE_JIT != 0 - seg_len = - aot_module->wasm_module->data_segments[seg_index]->data_length; - data = aot_module->wasm_module->data_segments[seg_index]->data; -#endif - } - else { - seg_len = aot_module->mem_init_data_list[seg_index]->byte_count; - data = aot_module->mem_init_data_list[seg_index]->bytes; - } + aot_module = (AOTModule *)module_inst->module; + seg_len = aot_module->mem_init_data_list[seg_index]->byte_count; + data = aot_module->mem_init_data_list[seg_index]->bytes; - if (!aot_validate_app_addr(module_inst, dst, len)) + if (!wasm_runtime_validate_app_addr((WASMModuleInstanceCommon *)module_inst, + dst, len)) return false; if ((uint64)offset + (uint64)len > seg_len) { @@ -2549,7 +1984,8 @@ aot_memory_init(AOTModuleInstance *module_inst, uint32 seg_index, uint32 offset, return false; } - maddr = aot_addr_app_to_native(module_inst, dst); + maddr = wasm_runtime_addr_app_to_native( + (WASMModuleInstanceCommon *)module_inst, dst); bh_memcpy_s(maddr, memory_inst->memory_data_size - dst, data + offset, len); return true; @@ -2558,20 +1994,11 @@ aot_memory_init(AOTModuleInstance *module_inst, uint32 seg_index, uint32 offset, bool aot_data_drop(AOTModuleInstance *module_inst, uint32 seg_index) { - AOTModule *aot_module = (AOTModule *)(module_inst->aot_module.ptr); + AOTModule *aot_module = (AOTModule *)module_inst->module; - if (aot_module->is_jit_mode) { -#if WASM_ENABLE_JIT != 0 - aot_module->wasm_module->data_segments[seg_index]->data_length = 0; - /* Currently we can't free the dropped data segment - as they are stored in wasm bytecode */ -#endif - } - else { - aot_module->mem_init_data_list[seg_index]->byte_count = 0; - /* Currently we can't free the dropped data segment - as the mem_init_data_count is a continuous array */ - } + aot_module->mem_init_data_list[seg_index]->byte_count = 0; + /* Currently we can't free the dropped data segment + as the mem_init_data_count is a continuous array */ return true; } #endif /* WASM_ENABLE_BULK_MEMORY */ @@ -2581,7 +2008,7 @@ bool aot_set_aux_stack(WASMExecEnv *exec_env, uint32 start_offset, uint32 size) { AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst; - AOTModule *module = (AOTModule *)module_inst->aot_module.ptr; + AOTModule *module = (AOTModule *)module_inst->module; uint32 stack_top_idx = module->aux_stack_top_global_index; uint32 data_end = module->aux_data_end; @@ -2597,8 +2024,7 @@ aot_set_aux_stack(WASMExecEnv *exec_env, uint32 start_offset, uint32 size) /* The aux stack top is a wasm global, set the initial value for the global */ uint32 global_offset = module->globals[stack_top_idx].data_offset; - uint8 *global_addr = - (uint8 *)module_inst->global_data.ptr + global_offset; + uint8 *global_addr = module_inst->global_data + global_offset; *(int32 *)global_addr = start_offset; /* The aux stack boundary is a constant value, @@ -2615,7 +2041,7 @@ bool aot_get_aux_stack(WASMExecEnv *exec_env, uint32 *start_offset, uint32 *size) { AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst; - AOTModule *module = (AOTModule *)module_inst->aot_module.ptr; + AOTModule *module = (AOTModule *)module_inst->module; /* The aux stack information is resolved in loader and store in module */ @@ -2738,31 +2164,31 @@ aot_get_module_inst_mem_consumption(const AOTModuleInstance *module_inst, mem_conspn->module_inst_struct_size = sizeof(AOTModuleInstance); mem_conspn->memories_size = - sizeof(AOTPointer) * module_inst->memory_count + sizeof(void *) * module_inst->memory_count + sizeof(AOTMemoryInstance) * module_inst->memory_count; for (i = 0; i < module_inst->memory_count; i++) { - AOTMemoryInstance *mem_inst = - ((AOTMemoryInstance **)module_inst->memories.ptr)[i]; + AOTMemoryInstance *mem_inst = module_inst->memories[i]; mem_conspn->memories_size += mem_inst->num_bytes_per_page * mem_inst->cur_page_count; - mem_conspn->app_heap_size = (uint8 *)mem_inst->heap_data_end.ptr - - (uint8 *)mem_inst->heap_data.ptr; + mem_conspn->app_heap_size = + mem_inst->heap_data_end - mem_inst->heap_data; /* size of app heap structure */ mem_conspn->memories_size += mem_allocator_get_heap_struct_size(); } - tbl_inst = module_inst->tables.ptr; + mem_conspn->tables_size += + sizeof(AOTTableInstance *) * module_inst->table_count; for (i = 0; i < module_inst->table_count; i++) { - mem_conspn->tables_size += offsetof(AOTTableInstance, data); + tbl_inst = module_inst->tables[i]; + mem_conspn->tables_size += offsetof(AOTTableInstance, elems); mem_conspn->tables_size += sizeof(uint32) * tbl_inst->max_size; - tbl_inst = aot_next_tbl_inst(tbl_inst); } /* func_ptrs and func_type_indexes */ mem_conspn->functions_size = (sizeof(void *) + sizeof(uint32)) - * (((AOTModule *)module_inst->aot_module.ptr)->import_func_count - + ((AOTModule *)module_inst->aot_module.ptr)->func_count); + * (((AOTModule *)module_inst->module)->import_func_count + + ((AOTModule *)module_inst->module)->func_count); mem_conspn->globals_size = module_inst->global_data_size; @@ -2783,7 +2209,7 @@ aot_get_module_inst_mem_consumption(const AOTModuleInstance *module_inst, void aot_drop_table_seg(AOTModuleInstance *module_inst, uint32 tbl_seg_idx) { - AOTModule *module = (AOTModule *)module_inst->aot_module.ptr; + AOTModule *module = (AOTModule *)module_inst->module; AOTTableInitData *tbl_seg = module->table_init_data_list[tbl_seg_idx]; tbl_seg->is_dropped = true; } @@ -2795,9 +2221,9 @@ aot_table_init(AOTModuleInstance *module_inst, uint32 tbl_idx, { AOTTableInstance *tbl_inst; AOTTableInitData *tbl_seg; - const AOTModule *module = module_inst->aot_module.ptr; + const AOTModule *module = (AOTModule *)module_inst->module; - tbl_inst = aot_get_table_inst(module_inst, tbl_idx); + tbl_inst = module_inst->tables[tbl_idx]; bh_assert(tbl_inst); tbl_seg = module->table_init_data_list[tbl_seg_idx]; @@ -2823,7 +2249,7 @@ aot_table_init(AOTModuleInstance *module_inst, uint32 tbl_idx, return; } - bh_memcpy_s((uint8 *)tbl_inst + offsetof(AOTTableInstance, data) + bh_memcpy_s((uint8 *)tbl_inst + offsetof(AOTTableInstance, elems) + dst_offset * sizeof(uint32), (tbl_inst->cur_size - dst_offset) * sizeof(uint32), tbl_seg->func_indexes + src_offset, length * sizeof(uint32)); @@ -2836,10 +2262,10 @@ aot_table_copy(AOTModuleInstance *module_inst, uint32 src_tbl_idx, { AOTTableInstance *src_tbl_inst, *dst_tbl_inst; - src_tbl_inst = aot_get_table_inst(module_inst, src_tbl_idx); + src_tbl_inst = module_inst->tables[src_tbl_idx]; bh_assert(src_tbl_inst); - dst_tbl_inst = aot_get_table_inst(module_inst, dst_tbl_idx); + dst_tbl_inst = module_inst->tables[dst_tbl_idx]; bh_assert(dst_tbl_inst); if ((uint64)dst_offset + length > dst_tbl_inst->cur_size @@ -2851,10 +2277,10 @@ aot_table_copy(AOTModuleInstance *module_inst, uint32 src_tbl_idx, /* if src_offset >= dst_offset, copy from front to back */ /* if src_offset < dst_offset, copy from back to front */ /* merge all together */ - bh_memmove_s((uint8 *)(dst_tbl_inst) + offsetof(AOTTableInstance, data) + bh_memmove_s((uint8 *)dst_tbl_inst + offsetof(AOTTableInstance, elems) + dst_offset * sizeof(uint32), (dst_tbl_inst->cur_size - dst_offset) * sizeof(uint32), - (uint8 *)(src_tbl_inst) + offsetof(AOTTableInstance, data) + (uint8 *)src_tbl_inst + offsetof(AOTTableInstance, elems) + src_offset * sizeof(uint32), length * sizeof(uint32)); } @@ -2865,7 +2291,7 @@ aot_table_fill(AOTModuleInstance *module_inst, uint32 tbl_idx, uint32 length, { AOTTableInstance *tbl_inst; - tbl_inst = aot_get_table_inst(module_inst, tbl_idx); + tbl_inst = module_inst->tables[tbl_idx]; bh_assert(tbl_inst); if (data_offset + length > tbl_inst->cur_size) { @@ -2874,7 +2300,7 @@ aot_table_fill(AOTModuleInstance *module_inst, uint32 tbl_idx, uint32 length, } for (; length != 0; data_offset++, length--) { - tbl_inst->data[data_offset] = val; + tbl_inst->elems[data_offset] = val; } } @@ -2885,7 +2311,7 @@ aot_table_grow(AOTModuleInstance *module_inst, uint32 tbl_idx, uint32 entry_count, i, orig_tbl_sz; AOTTableInstance *tbl_inst; - tbl_inst = aot_get_table_inst(module_inst, tbl_idx); + tbl_inst = module_inst->tables[tbl_idx]; if (!tbl_inst) { return (uint32)-1; } @@ -2907,7 +2333,7 @@ aot_table_grow(AOTModuleInstance *module_inst, uint32 tbl_idx, /* fill in */ for (i = 0; i < inc_entries; ++i) { - tbl_inst->data[tbl_inst->cur_size + i] = init_val; + tbl_inst->elems[tbl_inst->cur_size + i] = init_val; } tbl_inst->cur_size = entry_count; @@ -2947,7 +2373,7 @@ get_func_name_from_index(const AOTModuleInstance *module_inst, uint32 func_index) { const char *func_name = NULL; - AOTModule *module = module_inst->aot_module.ptr; + AOTModule *module = (AOTModule *)module_inst->module; #if WASM_ENABLE_CUSTOM_NAME_SECTION != 0 if ((func_name = @@ -2983,8 +2409,7 @@ aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index) #if WASM_ENABLE_PERF_PROFILING != 0 AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst; AOTFuncPerfProfInfo *func_perf_prof = - (AOTFuncPerfProfInfo *)module_inst->func_perf_profilings.ptr - + func_index; + module_inst->func_perf_profilings + func_index; #endif if (!frame) { @@ -3038,8 +2463,8 @@ aot_create_call_stack(struct WASMExecEnv *exec_env) } /* release previous stack frames and create new ones */ - if (!bh_vector_destroy(module_inst->frames.ptr) - || !bh_vector_init(module_inst->frames.ptr, n, sizeof(WASMCApiFrame), + if (!bh_vector_destroy(module_inst->frames) + || !bh_vector_init(module_inst->frames, n, sizeof(WASMCApiFrame), false)) { return false; } @@ -3054,8 +2479,8 @@ aot_create_call_stack(struct WASMExecEnv *exec_env) frame.func_name_wp = get_func_name_from_index(module_inst, cur_frame->func_index); - if (!bh_vector_append(module_inst->frames.ptr, &frame)) { - bh_vector_destroy(module_inst->frames.ptr); + if (!bh_vector_append(module_inst->frames, &frame)) { + bh_vector_destroy(module_inst->frames); return false; } @@ -3083,11 +2508,11 @@ aot_dump_call_stack(WASMExecEnv *exec_env, bool print, char *buf, uint32 len) * will be truncated */ char line_buf[256]; - if (!module_inst->frames.ptr) { + if (!module_inst->frames) { return 0; } - total_frames = (uint32)bh_vector_size(module_inst->frames.ptr); + total_frames = (uint32)bh_vector_size(module_inst->frames); if (total_frames == 0) { return 0; } @@ -3099,7 +2524,7 @@ aot_dump_call_stack(WASMExecEnv *exec_env, bool print, char *buf, uint32 len) WASMCApiFrame frame = { 0 }; uint32 line_length, i; - if (!bh_vector_get(module_inst->frames.ptr, n, &frame)) { + if (!bh_vector_get(module_inst->frames, n, &frame)) { return 0; } @@ -3138,8 +2563,8 @@ void aot_dump_perf_profiling(const AOTModuleInstance *module_inst) { AOTFuncPerfProfInfo *perf_prof = - (AOTFuncPerfProfInfo *)module_inst->func_perf_profilings.ptr; - AOTModule *module = (AOTModule *)module_inst->aot_module.ptr; + (AOTFuncPerfProfInfo *)module_inst->func_perf_profilings; + AOTModule *module = (AOTModule *)module_inst->module; uint32 total_func_count = module->import_func_count + module->func_count, i; const char *func_name; diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index 14e037e7..e5503ed1 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -10,34 +10,11 @@ #include "../common/wasm_runtime_common.h" #include "../interpreter/wasm_runtime.h" #include "../compilation/aot.h" -#if WASM_ENABLE_JIT != 0 -#include "../compilation/aot_llvm.h" -#endif #ifdef __cplusplus extern "C" { #endif -typedef enum AOTExceptionID { - EXCE_UNREACHABLE = 0, - EXCE_OUT_OF_MEMORY, - EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, - EXCE_INTEGER_OVERFLOW, - EXCE_INTEGER_DIVIDE_BY_ZERO, - EXCE_INVALID_CONVERSION_TO_INTEGER, - EXCE_INVALID_FUNCTION_TYPE_INDEX, - EXCE_INVALID_FUNCTION_INDEX, - EXCE_UNDEFINED_ELEMENT, - EXCE_UNINITIALIZED_ELEMENT, - EXCE_CALL_UNLINKED_IMPORT_FUNC, - EXCE_NATIVE_STACK_OVERFLOW, - EXCE_UNALIGNED_ATOMIC, - EXCE_AUX_STACK_OVERFLOW, - EXCE_AUX_STACK_UNDERFLOW, - EXCE_OUT_OF_BOUNDS_TABLE_ACCESS, - EXCE_NUM, -} AOTExceptionID; - typedef enum AOTSectionType { AOT_SECTION_TYPE_TARGET_INFO = 0, AOT_SECTION_TYPE_INIT_DATA = 1, @@ -171,7 +148,7 @@ typedef struct AOTModule { /* function info */ uint32 func_count; - /* point to AOTed/JITed functions */ + /* point to AOTed functions */ void **func_ptrs; /* function type indexes */ uint32 *func_type_indexes; @@ -182,24 +159,24 @@ typedef struct AOTModule { /* start function index, -1 denotes no start function */ uint32 start_func_index; - /* start function, point to AOTed/JITed function */ + /* start function, point to AOTed function */ void *start_function; uint32 malloc_func_index; uint32 free_func_index; uint32 retain_func_index; - /* AOTed code, NULL for JIT mode */ + /* AOTed code */ void *code; uint32 code_size; - /* literal for AOTed code, NULL for JIT mode */ + /* literal for AOTed code */ uint8 *literal; uint32 literal_size; #if defined(BH_PLATFORM_WINDOWS) /* extra plt data area for __ymm, __xmm and __real constants - in Windows platform, NULL for JIT mode */ + in Windows platform */ uint8 *extra_plt_data; uint32 extra_plt_data_size; uint32 ymm_plt_count; @@ -217,7 +194,7 @@ typedef struct AOTModule { #endif /* data sections in AOT object file, including .data, .rodata - * and .rodata.cstN. NULL for JIT mode. */ + and .rodata.cstN. */ AOTObjectDataSection *data_sections; uint32 data_section_count; @@ -244,18 +221,9 @@ typedef struct AOTModule { /* auxiliary stack size resolved */ uint32 aux_stack_size; - /* is jit mode or not */ - bool is_jit_mode; - /* is indirect mode or not */ bool is_indirect_mode; -#if WASM_ENABLE_JIT != 0 - WASMModule *wasm_module; - AOTCompContext *comp_ctx; - AOTCompData *comp_data; -#endif - #if WASM_ENABLE_LIBC_WASI != 0 WASIArguments wasi_args; bool import_wasi_api; @@ -274,130 +242,9 @@ typedef struct AOTModule { #endif } AOTModule; -typedef union { - uint64 _make_it_8_bytes_; - void *ptr; -} AOTPointer; - -typedef union { - uint64 u64; - uint32 u32[2]; -} MemBound; - -typedef struct AOTMemoryInstance { - uint32 module_type; - /* shared memory flag */ - bool is_shared; - - /* memory space info */ - uint32 num_bytes_per_page; - uint32 cur_page_count; - uint32 max_page_count; - uint32 memory_data_size; - AOTPointer memory_data; - AOTPointer memory_data_end; - - /* heap space info */ - AOTPointer heap_data; - AOTPointer heap_data_end; - AOTPointer heap_handle; - - /* boundary check constants for aot code */ - MemBound mem_bound_check_1byte; - MemBound mem_bound_check_2bytes; - MemBound mem_bound_check_4bytes; - MemBound mem_bound_check_8bytes; - MemBound mem_bound_check_16bytes; -} AOTMemoryInstance; - -typedef struct AOTTableInstance { - uint32 cur_size; - /* - * only grow in the range of [:max_size) - * if the table is growable, max_size equals to its declared maximum size - * otherwise, max_size equals to its declared minimum size - */ - uint32 max_size; - /* - * +------------------------------+ <--- data - * | ref.func[] or ref.extern[] - * +------------------------------+ - */ - uint32 data[1]; -} AOTTableInstance; - -typedef struct AOTModuleInstance { - uint32 module_type; - - /* memories */ - uint32 memory_count; - AOTPointer memories; - - /* global and table info */ - uint32 global_data_size; - /* - * the count of AOTTableInstance. - * it includes imported tables and local tables. - * - * TODO: for now we treate imported table like a local table - */ - uint32 table_count; - /* points to global_data */ - AOTPointer global_data; - /* points to AOTTableInstance[] */ - AOTPointer tables; - - /* function pointer array */ - AOTPointer func_ptrs; - /* function type indexes */ - AOTPointer func_type_indexes; - - /* export info */ - uint32 export_func_count; - uint32 export_global_count; - uint32 export_mem_count; - uint32 export_tab_count; - AOTPointer export_funcs; - AOTPointer export_globals; - AOTPointer export_memories; - AOTPointer export_tables; - - /* The exception buffer for current thread. */ - char cur_exception[128]; - /* The custom data that can be set/get by - * wasm_runtime_set_custom_data/wasm_runtime_get_custom_data */ - AOTPointer custom_data; - /* The AOT module */ - AOTPointer aot_module; - /* WASI context */ - AOTPointer wasi_ctx; - /* function performance profiling info list */ - AOTPointer func_perf_profilings; - /* stack frames, used in call stack dump and perf profiling */ - AOTPointer frames; - - AOTPointer exec_env_singleton; - - uint32 default_wasm_stack_size; - - /* reserved */ - uint32 reserved[9]; - - /* - * +------------------------------+ <-- memories.ptr - * | #0 AOTMemoryInstance - * +------------------------------+ <-- global_data.ptr - * | global data - * +------------------------------+ <-- tables.ptr - * | AOTTableInstance[table_count] - * +------------------------------+ - */ - union { - uint64 _make_it_8_byte_aligned_; - AOTMemoryInstance memory_instances[1]; - uint8 bytes[1]; - } global_table_data; -} AOTModuleInstance; +#define AOTMemoryInstance WASMMemoryInstance +#define AOTTableInstance WASMTableInstance +#define AOTModuleInstance WASMModuleInstance /* Target info, read from ELF header of object file */ typedef struct AOTTargetInfo { @@ -462,21 +309,6 @@ AOTModule * aot_load_from_sections(AOTSection *section_list, char *error_buf, uint32 error_buf_size); -#if WASM_ENABLE_JIT != 0 -/** - * Convert WASM module to AOT module - * - * @param wasm_module the WASM module to convert - * @param error_buf output of the error info - * @param error_buf_size the size of the error string - * - * @return return AOT module loaded, NULL if failed - */ -AOTModule * -aot_convert_wasm_module(WASMModule *wasm_module, char *error_buf, - uint32 error_buf_size); -#endif - /** * Unload a AOT module. * @@ -549,9 +381,6 @@ aot_create_exec_env_and_call_function(AOTModuleInstance *module_inst, AOTFunctionInstance *function, unsigned argc, uint32 argv[]); -bool -aot_create_exec_env_singleton(AOTModuleInstance *module_inst); - /** * Set AOT module instance exception with exception string * @@ -575,14 +404,6 @@ aot_set_exception_with_id(AOTModuleInstance *module_inst, uint32 id); const char * aot_get_exception(AOTModuleInstance *module_inst); -/** - * Clear exception info of the AOT module instance. - * - * @param module_inst the AOT module instance - */ -void -aot_clear_exception(AOTModuleInstance *module_inst); - uint32 aot_module_malloc(AOTModuleInstance *module_inst, uint32 size, void **p_native_addr); @@ -598,29 +419,6 @@ uint32 aot_module_dup_data(AOTModuleInstance *module_inst, const char *src, uint32 size); -bool -aot_validate_app_addr(AOTModuleInstance *module_inst, uint32 app_offset, - uint32 size); - -bool -aot_validate_native_addr(AOTModuleInstance *module_inst, void *native_ptr, - uint32 size); - -void * -aot_addr_app_to_native(AOTModuleInstance *module_inst, uint32 app_offset); - -uint32 -aot_addr_native_to_app(AOTModuleInstance *module_inst, void *native_ptr); - -bool -aot_get_app_addr_range(AOTModuleInstance *module_inst, uint32 app_offset, - uint32 *p_app_start_offset, uint32 *p_app_end_offset); - -bool -aot_get_native_addr_range(AOTModuleInstance *module_inst, uint8 *native_ptr, - uint8 **p_native_start_addr, - uint8 **p_native_end_addr); - bool aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count); @@ -635,6 +433,10 @@ bool aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, uint32 argc, uint32 *argv); +/** + * Check whether the app address and the buf is inside the linear memory, + * and convert the app address into native address + */ bool aot_check_app_addr_and_convert(AOTModuleInstance *module_inst, bool is_str, uint32 app_buf_addr, uint32 app_buf_size, @@ -666,16 +468,6 @@ bool aot_get_aux_stack(WASMExecEnv *exec_env, uint32 *start_offset, uint32 *size); #endif -#ifdef OS_ENABLE_HW_BOUND_CHECK -#ifndef BH_PLATFORM_WINDOWS -void -aot_signal_handler(WASMSignalInfo *sig_info); -#else -LONG -aot_exception_handler(WASMSignalInfo *sig_info); -#endif -#endif - void aot_get_module_mem_consumption(const AOTModule *module, WASMModuleMemConsumption *mem_conspn); @@ -707,9 +499,6 @@ aot_table_grow(AOTModuleInstance *module_inst, uint32 tbl_idx, uint32 inc_entries, uint32 init_val); #endif -AOTTableInstance * -aot_next_tbl_inst(const AOTTableInstance *tbl_inst); - bool aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index); diff --git a/core/iwasm/common/wasm_c_api.c b/core/iwasm/common/wasm_c_api.c index 52b6464a..4b1958fe 100644 --- a/core/iwasm/common/wasm_c_api.c +++ b/core/iwasm/common/wasm_c_api.c @@ -2,16 +2,20 @@ * Copyright (C) 2019 Intel Corporation. All rights reserved. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ - #include "wasm_c_api_internal.h" + +#include "bh_assert.h" #include "wasm_memory.h" -#include "wasm_runtime_common.h" #if WASM_ENABLE_INTERP != 0 #include "wasm_runtime.h" #endif #if WASM_ENABLE_AOT != 0 #include "aot_runtime.h" -#endif +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT == 0 +#include "aot.h" +#include "aot_llvm.h" +#endif /*WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT == 0*/ +#endif /*WASM_ENABLE_AOT != 0*/ #define ASSERT_NOT_IMPLEMENTED() bh_assert(!"not implemented") #define UNREACHABLE() bh_assert(!"unreachable") @@ -302,9 +306,11 @@ wasm_engine_new_internal(mem_alloc_type_t type, const MemAllocOption *opts) goto failed; } -#if BH_DEBUG != 0 +#ifndef NDEBUG + /*DEBUG*/ bh_log_set_verbose_level(5); #else + /*VERBOSE*/ bh_log_set_verbose_level(3); #endif @@ -1655,7 +1661,7 @@ wasm_trap_new_internal(WASMModuleInstanceCommon *inst_comm_rt, #if WASM_ENABLE_AOT != 0 if (inst_comm_rt->module_type == Wasm_Module_AoT) { - trap->frames = ((AOTModuleInstance *)inst_comm_rt)->frames.ptr; + trap->frames = ((AOTModuleInstance *)inst_comm_rt)->frames; } #endif #endif /* WASM_ENABLE_DUMP_CALL_STACK != 0 */ @@ -2379,6 +2385,61 @@ failed_exporttype_new: wasm_exporttype_vec_delete(out); } +#if WASM_ENABLE_JIT == 0 || WASM_ENABLE_LAZY_JIT != 0 +void +wasm_module_serialize(wasm_module_t *module, own wasm_byte_vec_t *out) +{ + (void)module; + (void)out; + LOG_ERROR("only supported serialization in JIT with eager compilation"); +} + +own wasm_module_t * +wasm_module_deserialize(wasm_store_t *module, const wasm_byte_vec_t *binary) +{ + (void)module; + (void)binary; + LOG_ERROR("only supported deserialization in JIT with eager compilation"); + return NULL; +} +#else + +extern uint8 * +aot_emit_aot_file_buf(AOTCompContext *comp_ctx, AOTCompData *comp_data, + uint32 *p_aot_file_size); +void +wasm_module_serialize(wasm_module_t *module, own wasm_byte_vec_t *out) +{ + wasm_module_ex_t *module_ex; + AOTCompContext *comp_ctx; + AOTCompData *comp_data; + uint8 *aot_file_buf = NULL; + uint32 aot_file_size = 0; + + if (!module || !out) + return; + + module_ex = module_to_module_ext(module); + comp_ctx = ((WASMModule *)(module_ex->module_comm_rt))->comp_ctx; + comp_data = ((WASMModule *)(module_ex->module_comm_rt))->comp_data; + bh_assert(comp_ctx != NULL && comp_data != NULL); + + aot_file_buf = aot_emit_aot_file_buf(comp_ctx, comp_data, &aot_file_size); + if (!aot_file_buf) + return; + + wasm_byte_vec_new(out, aot_file_size, (wasm_byte_t *)aot_file_buf); + wasm_runtime_free(aot_file_buf); + return; +} + +own wasm_module_t * +wasm_module_deserialize(wasm_store_t *store, const wasm_byte_vec_t *binary) +{ + return wasm_module_new(store, binary); +} +#endif + static wasm_func_t * wasm_func_new_basic(wasm_store_t *store, const wasm_functype_t *type, wasm_func_callback_t func_callback) @@ -2482,9 +2543,9 @@ wasm_func_new_internal(wasm_store_t *store, uint16 func_idx_rt, #if WASM_ENABLE_INTERP != 0 if (inst_comm_rt->module_type == Wasm_Module_Bytecode) { bh_assert(func_idx_rt - < ((WASMModuleInstance *)inst_comm_rt)->function_count); + < ((WASMModuleInstance *)inst_comm_rt)->e->function_count); WASMFunctionInstance *func_interp = - ((WASMModuleInstance *)inst_comm_rt)->functions + func_idx_rt; + ((WASMModuleInstance *)inst_comm_rt)->e->functions + func_idx_rt; type_rt = func_interp->is_import_func ? func_interp->u.func_import->func_type : func_interp->u.func->func_type; @@ -2496,7 +2557,7 @@ wasm_func_new_internal(wasm_store_t *store, uint16 func_idx_rt, /* use same index to trace the function type in AOTFuncType **func_types */ AOTModule *module_aot = - ((AOTModuleInstance *)inst_comm_rt)->aot_module.ptr; + (AOTModule *)((AOTModuleInstance *)inst_comm_rt)->module; if (func_idx_rt < module_aot->import_func_count) { type_rt = (module_aot->import_funcs + func_idx_rt)->func_type; } @@ -2605,6 +2666,7 @@ params_to_argv(const wasm_val_vec_t *params, } if (!params || !params->num_elems || !params->size || !params->data) { + LOG_ERROR("the parameter params is invalid"); return false; } @@ -2662,6 +2724,7 @@ argv_to_results(const uint32 *argv, const wasm_valtype_vec_t *result_defs, } if (!results || !results->size || !results->data) { + LOG_ERROR("the parameter results is invalid"); return false; } @@ -2748,7 +2811,7 @@ wasm_func_call(const wasm_func_t *func, const wasm_val_vec_t *params, #if WASM_ENABLE_INTERP != 0 if (func->inst_comm_rt->module_type == Wasm_Module_Bytecode) { - func_comm_rt = ((WASMModuleInstance *)func->inst_comm_rt)->functions + func_comm_rt = ((WASMModuleInstance *)func->inst_comm_rt)->e->functions + func->func_idx_rt; } #endif @@ -2758,7 +2821,7 @@ wasm_func_call(const wasm_func_t *func, const wasm_val_vec_t *params, if (!(func_comm_rt = func->func_comm_rt)) { AOTModuleInstance *inst_aot = (AOTModuleInstance *)func->inst_comm_rt; - AOTModule *module_aot = (AOTModule *)inst_aot->aot_module.ptr; + AOTModule *module_aot = (AOTModule *)inst_aot->module; uint32 export_i = 0, export_func_j = 0; for (; export_i < module_aot->export_count; ++export_i) { @@ -2766,7 +2829,7 @@ wasm_func_call(const wasm_func_t *func, const wasm_val_vec_t *params, if (export->kind == EXPORT_KIND_FUNC) { if (export->index == func->func_idx_rt) { func_comm_rt = - (AOTFunctionInstance *)inst_aot->export_funcs.ptr + (AOTFunctionInstance *)inst_aot->export_functions + export_func_j; ((wasm_func_t *)func)->func_comm_rt = func_comm_rt; break; @@ -2788,6 +2851,7 @@ wasm_func_call(const wasm_func_t *func, const wasm_val_vec_t *params, param_count = wasm_func_param_arity(func); result_count = wasm_func_result_arity(func); + alloc_count = (param_count > result_count) ? param_count : result_count; if (alloc_count > (size_t)sizeof(argv_buf) / sizeof(uint64)) { if (!(argv = malloc_internal(sizeof(uint64) * alloc_count))) { @@ -2968,7 +3032,7 @@ interp_global_set(const WASMModuleInstance *inst_interp, uint16 global_idx_rt, const wasm_val_t *v) { const WASMGlobalInstance *global_interp = - inst_interp->globals + global_idx_rt; + inst_interp->e->globals + global_idx_rt; uint8 val_type_rt = global_interp->type; #if WASM_ENABLE_MULTI_MODULE != 0 uint8 *data = global_interp->import_global_inst @@ -2987,7 +3051,7 @@ static bool interp_global_get(const WASMModuleInstance *inst_interp, uint16 global_idx_rt, wasm_val_t *out) { - WASMGlobalInstance *global_interp = inst_interp->globals + global_idx_rt; + WASMGlobalInstance *global_interp = inst_interp->e->globals + global_idx_rt; uint8 val_type_rt = global_interp->type; #if WASM_ENABLE_MULTI_MODULE != 0 uint8 *data = global_interp->import_global_inst @@ -3007,7 +3071,7 @@ static bool aot_global_set(const AOTModuleInstance *inst_aot, uint16 global_idx_rt, const wasm_val_t *v) { - AOTModule *module_aot = inst_aot->aot_module.ptr; + AOTModule *module_aot = (AOTModule *)inst_aot->module; uint8 val_type_rt = 0; uint32 data_offset = 0; void *data = NULL; @@ -3025,7 +3089,7 @@ aot_global_set(const AOTModuleInstance *inst_aot, uint16 global_idx_rt, .type; } - data = (void *)((uint8 *)inst_aot->global_data.ptr + data_offset); + data = (void *)(inst_aot->global_data + data_offset); return wasm_val_to_rt_val((WASMModuleInstanceCommon *)inst_aot, val_type_rt, v, data); } @@ -3034,7 +3098,7 @@ static bool aot_global_get(const AOTModuleInstance *inst_aot, uint16 global_idx_rt, wasm_val_t *out) { - AOTModule *module_aot = inst_aot->aot_module.ptr; + AOTModule *module_aot = (AOTModule *)inst_aot->module; uint8 val_type_rt = 0; uint32 data_offset = 0; uint8 *data = NULL; @@ -3052,7 +3116,7 @@ aot_global_get(const AOTModuleInstance *inst_aot, uint16 global_idx_rt, .type; } - data = (uint8 *)inst_aot->global_data.ptr + data_offset; + data = inst_aot->global_data + data_offset; return rt_val_to_wasm_val(data, val_type_rt, out); } #endif @@ -3149,7 +3213,7 @@ wasm_global_new_internal(wasm_store_t *store, uint16 global_idx_rt, #if WASM_ENABLE_INTERP != 0 if (inst_comm_rt->module_type == Wasm_Module_Bytecode) { WASMGlobalInstance *global_interp = - ((WASMModuleInstance *)inst_comm_rt)->globals + global_idx_rt; + ((WASMModuleInstance *)inst_comm_rt)->e->globals + global_idx_rt; val_type_rt = global_interp->type; is_mutable = global_interp->is_mutable; init = true; @@ -3159,7 +3223,7 @@ wasm_global_new_internal(wasm_store_t *store, uint16 global_idx_rt, #if WASM_ENABLE_AOT != 0 if (inst_comm_rt->module_type == Wasm_Module_AoT) { AOTModuleInstance *inst_aot = (AOTModuleInstance *)inst_comm_rt; - AOTModule *module_aot = inst_aot->aot_module.ptr; + AOTModule *module_aot = (AOTModule *)inst_aot->module; init = true; @@ -3257,7 +3321,6 @@ wasm_table_new_internal(wasm_store_t *store, uint16 table_idx_rt, wasm_table_t *table = NULL; uint8 val_type_rt = 0; uint32 init_size = 0, max_size = 0; - bool init_flag = false; bh_assert(singleton_engine); @@ -3272,46 +3335,12 @@ wasm_table_new_internal(wasm_store_t *store, uint16 table_idx_rt, table->store = store; table->kind = WASM_EXTERN_TABLE; -#if WASM_ENABLE_INTERP != 0 - if (inst_comm_rt->module_type == Wasm_Module_Bytecode) { - WASMTableInstance *table_interp = - ((WASMModuleInstance *)inst_comm_rt)->tables[table_idx_rt]; - val_type_rt = table_interp->elem_type; - init_size = table_interp->cur_size; - max_size = table_interp->max_size; - init_flag = true; - } -#endif - -#if WASM_ENABLE_AOT != 0 - if (inst_comm_rt->module_type == Wasm_Module_AoT) { - AOTModuleInstance *inst_aot = (AOTModuleInstance *)inst_comm_rt; - AOTModule *module_aot = (AOTModule *)inst_aot->aot_module.ptr; - - if (table_idx_rt < module_aot->import_table_count) { - AOTImportTable *table_aot = - module_aot->import_tables + table_idx_rt; - val_type_rt = table_aot->elem_type; - init_size = table_aot->table_init_size; - max_size = table_aot->table_max_size; - } - else { - AOTTable *table_aot = - module_aot->tables - + (table_idx_rt - module_aot->import_table_count); - val_type_rt = table_aot->elem_type; - init_size = table_aot->table_init_size; - max_size = table_aot->table_max_size; - } - init_flag = true; - } -#endif - - /* - * a wrong combination of module filetype and compilation flags - * leads to below branch - */ - if (!init_flag) { + if (!wasm_runtime_get_table_inst_elem_type( + inst_comm_rt, table_idx_rt, &val_type_rt, &init_size, &max_size)) { + /* + * a wrong combination of module filetype and compilation flags + * leads to below branch + */ goto failed; } @@ -3400,19 +3429,18 @@ wasm_table_get(const wasm_table_t *table, wasm_table_size_t index) if (index >= table_interp->cur_size) { return NULL; } - ref_idx = ((uint32 *)table_interp->base_addr)[index]; + ref_idx = table_interp->elems[index]; } #endif #if WASM_ENABLE_AOT != 0 if (table->inst_comm_rt->module_type == Wasm_Module_AoT) { AOTModuleInstance *inst_aot = (AOTModuleInstance *)table->inst_comm_rt; - AOTTableInstance *table_aot = - (AOTTableInstance *)inst_aot->tables.ptr + table->table_idx_rt; + AOTTableInstance *table_aot = inst_aot->tables[table->table_idx_rt]; if (index >= table_aot->cur_size) { return NULL; } - ref_idx = table_aot->data[index]; + ref_idx = table_aot->elems[index]; } #endif @@ -3472,24 +3500,23 @@ wasm_table_set(wasm_table_t *table, wasm_table_size_t index, return false; } - p_ref_idx = ((uint32 *)table_interp->base_addr) + index; + p_ref_idx = table_interp->elems + index; function_count = - ((WASMModuleInstance *)table->inst_comm_rt)->function_count; + ((WASMModuleInstance *)table->inst_comm_rt)->e->function_count; } #endif #if WASM_ENABLE_AOT != 0 if (table->inst_comm_rt->module_type == Wasm_Module_AoT) { AOTModuleInstance *inst_aot = (AOTModuleInstance *)table->inst_comm_rt; - AOTModule *module_aot = (AOTModule *)inst_aot->aot_module.ptr; - AOTTableInstance *table_aot = - (AOTTableInstance *)inst_aot->tables.ptr + table->table_idx_rt; + AOTModule *module_aot = (AOTModule *)inst_aot->module; + AOTTableInstance *table_aot = inst_aot->tables[table->table_idx_rt]; if (index >= table_aot->cur_size) { return false; } - p_ref_idx = table_aot->data + index; + p_ref_idx = table_aot->elems + index; function_count = module_aot->func_count; } #endif @@ -3545,7 +3572,7 @@ wasm_table_size(const wasm_table_t *table) #if WASM_ENABLE_AOT != 0 if (table->inst_comm_rt->module_type == Wasm_Module_AoT) { AOTModuleInstance *inst_aot = (AOTModuleInstance *)table->inst_comm_rt; - AOTModule *module_aot = (AOTModule *)inst_aot->aot_module.ptr; + AOTModule *module_aot = (AOTModule *)inst_aot->module; if (table->table_idx_rt < module_aot->import_table_count) { AOTImportTable *table_aot = @@ -3660,7 +3687,7 @@ wasm_memory_new_internal(wasm_store_t *store, uint16 memory_idx_rt, #if WASM_ENABLE_AOT != 0 if (inst_comm_rt->module_type == Wasm_Module_AoT) { AOTModuleInstance *inst_aot = (AOTModuleInstance *)inst_comm_rt; - AOTModule *module_aot = (AOTModule *)(inst_aot->aot_module.ptr); + AOTModule *module_aot = (AOTModule *)inst_aot->module; if (memory_idx_rt < module_aot->import_memory_count) { min_pages = module_aot->import_memories->mem_init_page_count; @@ -3744,8 +3771,8 @@ wasm_memory_data(wasm_memory_t *memory) AOTModuleInstance *module_inst = (AOTModuleInstance *)module_inst_comm; AOTMemoryInstance *memory_inst = ((AOTMemoryInstance **) - module_inst->memories.ptr)[memory->memory_idx_rt]; - return (byte_t *)memory_inst->memory_data.ptr; + module_inst->memories)[memory->memory_idx_rt]; + return (byte_t *)memory_inst->memory_data; } #endif @@ -3781,7 +3808,7 @@ wasm_memory_data_size(const wasm_memory_t *memory) AOTModuleInstance *module_inst = (AOTModuleInstance *)module_inst_comm; AOTMemoryInstance *memory_inst = ((AOTMemoryInstance **) - module_inst->memories.ptr)[memory->memory_idx_rt]; + module_inst->memories)[memory->memory_idx_rt]; return memory_inst->cur_page_count * memory_inst->num_bytes_per_page; } #endif @@ -3818,7 +3845,7 @@ wasm_memory_size(const wasm_memory_t *memory) AOTModuleInstance *module_inst = (AOTModuleInstance *)module_inst_comm; AOTMemoryInstance *memory_inst = ((AOTMemoryInstance **) - module_inst->memories.ptr)[memory->memory_idx_rt]; + module_inst->memories)[memory->memory_idx_rt]; return memory_inst->cur_page_count; } #endif @@ -4197,7 +4224,7 @@ aot_process_export(wasm_store_t *store, const AOTModuleInstance *inst_aot, bh_assert(store && inst_aot && externals); - module_aot = (AOTModule *)inst_aot->aot_module.ptr; + module_aot = (AOTModule *)inst_aot->module; bh_assert(module_aot); for (i = 0; i < module_aot->export_count; ++i) { @@ -4420,8 +4447,9 @@ wasm_instance_new_with_args(wasm_store_t *store, const wasm_module_t *module, uint32 export_cnt = ((AOTModuleInstance *)instance->inst_comm_rt)->export_func_count + ((AOTModuleInstance *)instance->inst_comm_rt)->export_global_count - + ((AOTModuleInstance *)instance->inst_comm_rt)->export_tab_count - + ((AOTModuleInstance *)instance->inst_comm_rt)->export_mem_count; + + ((AOTModuleInstance *)instance->inst_comm_rt)->export_table_count + + ((AOTModuleInstance *)instance->inst_comm_rt) + ->export_memory_count; INIT_VEC(instance->exports, wasm_extern_vec_new_uninitialized, export_cnt); diff --git a/core/iwasm/common/wasm_exec_env.c b/core/iwasm/common/wasm_exec_env.c index ae05cce7..cd8b5708 100644 --- a/core/iwasm/common/wasm_exec_env.c +++ b/core/iwasm/common/wasm_exec_env.c @@ -65,7 +65,7 @@ wasm_exec_env_create_internal(struct WASMModuleInstanceCommon *module_inst, #if WASM_ENABLE_AOT != 0 if (module_inst->module_type == Wasm_Module_AoT) { AOTModuleInstance *i = (AOTModuleInstance *)module_inst; - AOTModule *m = (AOTModule *)i->aot_module.ptr; + AOTModule *m = (AOTModule *)i->module; exec_env->native_symbol = m->native_symbol_list; } #endif @@ -135,7 +135,7 @@ wasm_exec_env_create(struct WASMModuleInstanceCommon *module_inst, /* Set the aux_stack_boundary and aux_stack_bottom */ if (module_inst->module_type == Wasm_Module_AoT) { AOTModule *module = - (AOTModule *)(((AOTModuleInstance *)module_inst)->aot_module.ptr); + (AOTModule *)((AOTModuleInstance *)module_inst)->module; exec_env->aux_stack_bottom.bottom = module->aux_stack_bottom; exec_env->aux_stack_boundary.boundary = module->aux_stack_bottom - module->aux_stack_size; diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 49a9b086..ffe21857 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -4,6 +4,7 @@ */ #include "wasm_runtime_common.h" +#include "../interpreter/wasm_runtime.h" #include "bh_platform.h" #include "mem_alloc.h" @@ -138,6 +139,9 @@ wasm_runtime_free_internal(void *ptr) { if (!ptr) { LOG_WARNING("warning: wasm_runtime_free with NULL pointer\n"); +#if BH_ENABLE_GC_VERIFY != 0 + exit(-1); +#endif return; } @@ -160,6 +164,9 @@ wasm_runtime_malloc(unsigned int size) LOG_WARNING("warning: wasm_runtime_malloc with size zero\n"); /* At lease alloc 1 byte to avoid malloc failed */ size = 1; +#if BH_ENABLE_GC_VERIFY != 0 + exit(-1); +#endif } return wasm_runtime_malloc_internal(size); @@ -185,3 +192,446 @@ wasm_runtime_get_mem_alloc_info(mem_alloc_info_t *mem_alloc_info) } return false; } + +bool +wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, + uint32 app_offset, uint32 size) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + goto fail; + } + + /* integer overflow check */ + if (app_offset > UINT32_MAX - size) { + goto fail; + } + + if (app_offset + size <= memory_inst->memory_data_size) { + return true; + } + +fail: + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; +} + +bool +wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm, + uint32 app_str_offset) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + uint32 app_end_offset; + char *str, *str_end; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + if (!wasm_runtime_get_app_addr_range(module_inst_comm, app_str_offset, NULL, + &app_end_offset)) + goto fail; + + str = wasm_runtime_addr_app_to_native(module_inst_comm, app_str_offset); + str_end = str + (app_end_offset - app_str_offset); + while (str < str_end && *str != '\0') + str++; + if (str == str_end) + goto fail; + + return true; +fail: + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; +} + +bool +wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm, + void *native_ptr, uint32 size) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + uint8 *addr = (uint8 *)native_ptr; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + goto fail; + } + + /* integer overflow check */ + if ((uintptr_t)addr > UINTPTR_MAX - size) { + goto fail; + } + + if (memory_inst->memory_data <= addr + && addr + size <= memory_inst->memory_data_end) { + return true; + } + +fail: + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; +} + +void * +wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, + uint32 app_offset) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + uint8 *addr; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + return NULL; + } + + addr = memory_inst->memory_data + app_offset; + + if (memory_inst->memory_data <= addr && addr < memory_inst->memory_data_end) + return addr; + + return NULL; +} + +uint32 +wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, + void *native_ptr) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + uint8 *addr = (uint8 *)native_ptr; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + return 0; + } + + if (memory_inst->memory_data <= addr && addr < memory_inst->memory_data_end) + return (uint32)(addr - memory_inst->memory_data); + + return 0; +} + +bool +wasm_runtime_get_app_addr_range(WASMModuleInstanceCommon *module_inst_comm, + uint32 app_offset, uint32 *p_app_start_offset, + uint32 *p_app_end_offset) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + uint32 memory_data_size; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + return false; + } + + memory_data_size = memory_inst->memory_data_size; + + if (app_offset < memory_data_size) { + if (p_app_start_offset) + *p_app_start_offset = 0; + if (p_app_end_offset) + *p_app_end_offset = memory_data_size; + return true; + } + + return false; +} + +bool +wasm_runtime_get_native_addr_range(WASMModuleInstanceCommon *module_inst_comm, + uint8 *native_ptr, + uint8 **p_native_start_addr, + uint8 **p_native_end_addr) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + uint8 *addr = (uint8 *)native_ptr; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + return false; + } + + if (memory_inst->memory_data <= addr + && addr < memory_inst->memory_data_end) { + if (p_native_start_addr) + *p_native_start_addr = memory_inst->memory_data; + if (p_native_end_addr) + *p_native_end_addr = memory_inst->memory_data_end; + return true; + } + + return false; +} + +bool +wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, + uint32 app_buf_addr, uint32 app_buf_size, + void **p_native_addr) +{ + WASMMemoryInstance *memory_inst = wasm_get_default_memory(module_inst); + uint8 *native_addr; + + if (!memory_inst) { + goto fail; + } + + native_addr = memory_inst->memory_data + app_buf_addr; + + /* No need to check the app_offset and buf_size if memory access + boundary check with hardware trap is enabled */ +#ifndef OS_ENABLE_HW_BOUND_CHECK + if (app_buf_addr >= memory_inst->memory_data_size) { + goto fail; + } + + if (!is_str) { + if (app_buf_size > memory_inst->memory_data_size - app_buf_addr) { + goto fail; + } + } + else { + const char *str, *str_end; + + /* The whole string must be in the linear memory */ + str = (const char *)native_addr; + str_end = (const char *)memory_inst->memory_data_end; + while (str < str_end && *str != '\0') + str++; + if (str == str_end) + goto fail; + } +#endif + + *p_native_addr = (void *)native_addr; + return true; +fail: + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; +} + +WASMMemoryInstance * +wasm_get_default_memory(WASMModuleInstance *module_inst) +{ + if (module_inst->memories) + return module_inst->memories[0]; + else + return NULL; +} + +#ifndef OS_ENABLE_HW_BOUND_CHECK +bool +wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) +{ + WASMMemoryInstance *memory = wasm_get_default_memory(module); + uint8 *memory_data_old, *memory_data_new, *heap_data_old; + uint32 num_bytes_per_page, heap_size, total_size_old; + uint32 cur_page_count, max_page_count, total_page_count; + uint64 total_size_new; + bool ret = true; + + if (!memory) + return false; + + heap_data_old = memory->heap_data; + heap_size = (uint32)(memory->heap_data_end - memory->heap_data); + + memory_data_old = memory->memory_data; + total_size_old = memory->memory_data_size; + + num_bytes_per_page = memory->num_bytes_per_page; + cur_page_count = memory->cur_page_count; + max_page_count = memory->max_page_count; + total_page_count = inc_page_count + cur_page_count; + total_size_new = num_bytes_per_page * (uint64)total_page_count; + + if (inc_page_count <= 0) + /* No need to enlarge memory */ + return true; + + if (total_page_count < cur_page_count /* integer overflow */ + || total_page_count > max_page_count) { + return false; + } + + bh_assert(total_size_new <= 4 * (uint64)BH_GB); + if (total_size_new > UINT32_MAX) { + /* Resize to 1 page with size 4G-1 */ + num_bytes_per_page = UINT32_MAX; + total_page_count = max_page_count = 1; + total_size_new = UINT32_MAX; + } + +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (memory->is_shared) { + memory->num_bytes_per_page = num_bytes_per_page; + memory->cur_page_count = total_page_count; + memory->max_page_count = max_page_count; + /* No need to update memory->memory_data_size as it is + initialized with the maximum memory data size for + shared memory */ + return true; + } +#endif + + if (heap_size > 0) { + if (mem_allocator_is_heap_corrupted(memory->heap_handle)) { + wasm_runtime_show_app_heap_corrupted_prompt(); + return false; + } + } + + if (!(memory_data_new = + wasm_runtime_realloc(memory_data_old, (uint32)total_size_new))) { + if (!(memory_data_new = wasm_runtime_malloc((uint32)total_size_new))) { + return false; + } + if (memory_data_old) { + bh_memcpy_s(memory_data_new, (uint32)total_size_new, + memory_data_old, total_size_old); + wasm_runtime_free(memory_data_old); + } + } + + memset(memory_data_new + total_size_old, 0, + (uint32)total_size_new - total_size_old); + + if (heap_size > 0) { + if (mem_allocator_migrate(memory->heap_handle, + (char *)heap_data_old + + (memory_data_new - memory_data_old), + heap_size) + != 0) { + /* Don't return here as memory->memory_data is obsolete and + must be updated to be correctly used later. */ + ret = false; + } + } + + memory->heap_data = memory_data_new + (heap_data_old - memory_data_old); + memory->heap_data_end = memory->heap_data + heap_size; + + memory->num_bytes_per_page = num_bytes_per_page; + memory->cur_page_count = total_page_count; + memory->max_page_count = max_page_count; + memory->memory_data_size = (uint32)total_size_new; + + memory->memory_data = memory_data_new; + memory->memory_data_end = memory_data_new + (uint32)total_size_new; + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 || WASM_ENABLE_AOT != 0 +#if UINTPTR_MAX == UINT64_MAX + memory->mem_bound_check_1byte.u64 = total_size_new - 1; + memory->mem_bound_check_2bytes.u64 = total_size_new - 2; + memory->mem_bound_check_4bytes.u64 = total_size_new - 4; + memory->mem_bound_check_8bytes.u64 = total_size_new - 8; + memory->mem_bound_check_16bytes.u64 = total_size_new - 16; +#else + memory->mem_bound_check_1byte.u32[0] = (uint32)total_size_new - 1; + memory->mem_bound_check_2bytes.u32[0] = (uint32)total_size_new - 2; + memory->mem_bound_check_4bytes.u32[0] = (uint32)total_size_new - 4; + memory->mem_bound_check_8bytes.u32[0] = (uint32)total_size_new - 8; + memory->mem_bound_check_16bytes.u32[0] = (uint32)total_size_new - 16; +#endif +#endif + + return ret; +} +#else +bool +wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) +{ + WASMMemoryInstance *memory = wasm_get_default_memory(module); + uint32 num_bytes_per_page, total_size_old; + uint32 cur_page_count, max_page_count, total_page_count; + uint64 total_size_new; + + if (!memory) + return false; + + num_bytes_per_page = memory->num_bytes_per_page; + cur_page_count = memory->cur_page_count; + max_page_count = memory->max_page_count; + total_size_old = num_bytes_per_page * cur_page_count; + total_page_count = inc_page_count + cur_page_count; + total_size_new = num_bytes_per_page * (uint64)total_page_count; + + if (inc_page_count <= 0) + /* No need to enlarge memory */ + return true; + + if (total_page_count < cur_page_count /* integer overflow */ + || total_page_count > max_page_count) { + return false; + } + + bh_assert(total_size_new <= 4 * (uint64)BH_GB); + if (total_size_new > UINT32_MAX) { + /* Resize to 1 page with size 4G-1 */ + num_bytes_per_page = UINT32_MAX; + total_page_count = max_page_count = 1; + total_size_new = UINT32_MAX; + } + +#ifdef BH_PLATFORM_WINDOWS + if (!os_mem_commit(memory->memory_data_end, + (uint32)total_size_new - total_size_old, + MMAP_PROT_READ | MMAP_PROT_WRITE)) { + return false; + } +#endif + + if (os_mprotect(memory->memory_data_end, + (uint32)total_size_new - total_size_old, + MMAP_PROT_READ | MMAP_PROT_WRITE) + != 0) { +#ifdef BH_PLATFORM_WINDOWS + os_mem_decommit(memory->memory_data_end, + (uint32)total_size_new - total_size_old); +#endif + return false; + } + + /* The increased pages are filled with zero by the OS when os_mmap, + no need to memset it again here */ + + memory->num_bytes_per_page = num_bytes_per_page; + memory->cur_page_count = total_page_count; + memory->max_page_count = max_page_count; + memory->memory_data_size = (uint32)total_size_new; + memory->memory_data_end = memory->memory_data + (uint32)total_size_new; + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 || WASM_ENABLE_AOT != 0 + memory->mem_bound_check_1byte.u64 = total_size_new - 1; + memory->mem_bound_check_2bytes.u64 = total_size_new - 2; + memory->mem_bound_check_4bytes.u64 = total_size_new - 4; + memory->mem_bound_check_8bytes.u64 = total_size_new - 8; + memory->mem_bound_check_16bytes.u64 = total_size_new - 16; +#endif + + return true; +} +#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index 409ea0ff..cd4ecb70 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -135,43 +135,117 @@ static os_thread_local_attribute WASMExecEnv *exec_env_tls = NULL; static void runtime_signal_handler(void *sig_addr) { - WASMModuleInstanceCommon *module_inst; - WASMSignalInfo sig_info; + WASMModuleInstance *module_inst; + WASMMemoryInstance *memory_inst; + WASMJmpBuf *jmpbuf_node; + uint8 *mapped_mem_start_addr = NULL; + uint8 *mapped_mem_end_addr = NULL; + uint8 *stack_min_addr; + uint32 page_size; + uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; - sig_info.exec_env_tls = exec_env_tls; - sig_info.sig_addr = sig_addr; - if (exec_env_tls) { - module_inst = exec_env_tls->module_inst; -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - wasm_signal_handler(&sig_info); -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - aot_signal_handler(&sig_info); -#endif + /* Check whether current thread is running wasm function */ + if (exec_env_tls && exec_env_tls->handle == os_self_thread() + && (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) { + /* Get mapped mem info of current instance */ + module_inst = (WASMModuleInstance *)exec_env_tls->module_inst; + /* Get the default memory instance */ + memory_inst = wasm_get_default_memory(module_inst); + if (memory_inst) { + mapped_mem_start_addr = memory_inst->memory_data; + mapped_mem_end_addr = memory_inst->memory_data + 8 * (uint64)BH_GB; + } + + /* Get stack info of current thread */ + page_size = os_getpagesize(); + stack_min_addr = os_thread_get_stack_boundary(); + + if (memory_inst + && (mapped_mem_start_addr <= (uint8 *)sig_addr + && (uint8 *)sig_addr < mapped_mem_end_addr)) { + /* The address which causes segmentation fault is inside + the memory instance's guard regions */ + wasm_set_exception(module_inst, "out of bounds memory access"); + os_longjmp(jmpbuf_node->jmpbuf, 1); + } + else if (stack_min_addr - page_size <= (uint8 *)sig_addr + && (uint8 *)sig_addr + < stack_min_addr + page_size * guard_page_count) { + /* The address which causes segmentation fault is inside + native thread's guard page */ + wasm_set_exception(module_inst, "native stack overflow"); + os_longjmp(jmpbuf_node->jmpbuf, 1); + } } } #else static LONG runtime_exception_handler(EXCEPTION_POINTERS *exce_info) { - WASMModuleInstanceCommon *module_inst; - WASMSignalInfo sig_info; + PEXCEPTION_RECORD ExceptionRecord = exce_info->ExceptionRecord; + uint8 *sig_addr = (uint8 *)ExceptionRecord->ExceptionInformation[1]; + WASMModuleInstance *module_inst; + WASMMemoryInstance *memory_inst; + WASMJmpBuf *jmpbuf_node; + uint8 *mapped_mem_start_addr = NULL; + uint8 *mapped_mem_end_addr = NULL; + uint32 page_size = os_getpagesize(); - sig_info.exec_env_tls = exec_env_tls; - sig_info.exce_info = exce_info; - if (exec_env_tls) { - module_inst = exec_env_tls->module_inst; -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - return wasm_exception_handler(&sig_info); -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - return aot_exception_handler(&sig_info); -#endif + if (exec_env_tls && exec_env_tls->handle == os_self_thread() + && (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) { + module_inst = (WASMModuleInstance *)exec_env_tls->module_inst; + if (ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { + /* Get the default memory instance */ + memory_inst = wasm_get_default_memory(module_inst); + if (memory_inst) { + mapped_mem_start_addr = memory_inst->memory_data; + mapped_mem_end_addr = + memory_inst->memory_data + 8 * (uint64)BH_GB; + if (mapped_mem_start_addr <= (uint8 *)sig_addr + && (uint8 *)sig_addr < mapped_mem_end_addr) { + /* The address which causes segmentation fault is inside + the memory instance's guard regions. + Set exception and let the wasm func continue to run, when + the wasm func returns, the caller will check whether the + exception is thrown and return to runtime. */ + wasm_set_exception(module_inst, + "out of bounds memory access"); + if (module_inst->module_type == Wasm_Module_Bytecode) { + /* Continue to search next exception handler for + interpreter mode as it can be caught by + `__try { .. } __except { .. }` sentences in + wasm_runtime.c */ + return EXCEPTION_CONTINUE_SEARCH; + } + else { + /* Skip current instruction and continue to run for + AOT mode. TODO: implement unwind support for AOT + code in Windows platform */ + exce_info->ContextRecord->Rip++; + return EXCEPTION_CONTINUE_EXECUTION; + } + } + } + } + else if (ExceptionRecord->ExceptionCode == EXCEPTION_STACK_OVERFLOW) { + /* Set stack overflow exception and let the wasm func continue + to run, when the wasm func returns, the caller will check + whether the exception is thrown and return to runtime, and + the damaged stack will be recovered by _resetstkoflw(). */ + wasm_set_exception(module_inst, "native stack overflow"); + if (module_inst->module_type == Wasm_Module_Bytecode) { + return EXCEPTION_CONTINUE_SEARCH; + } + else { + return EXCEPTION_CONTINUE_EXECUTION; + } + } } + + os_printf("Unhandled exception thrown: exception code: 0x%lx, " + "exception address: %p, exception information: %p\n", + ExceptionRecord->ExceptionCode, ExceptionRecord->ExceptionAddress, + sig_addr); return EXCEPTION_CONTINUE_SEARCH; } #endif /* end of BH_PLATFORM_WINDOWS */ @@ -912,22 +986,7 @@ wasm_runtime_load(uint8 *buf, uint32 size, char *error_buf, WASMModuleCommon *module_common = NULL; if (get_package_type(buf, size) == Wasm_Module_Bytecode) { -#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_JIT != 0 - AOTModule *aot_module; - WASMModule *module = wasm_load(buf, size, error_buf, error_buf_size); - if (!module) - return NULL; - - if (!(aot_module = - aot_convert_wasm_module(module, error_buf, error_buf_size))) { - wasm_unload(module); - return NULL; - } - - module_common = (WASMModuleCommon *)aot_module; - return register_module_with_null_name(module_common, error_buf, - error_buf_size); -#elif WASM_ENABLE_INTERP != 0 +#if WASM_ENABLE_INTERP != 0 module_common = (WASMModuleCommon *)wasm_load(buf, size, error_buf, error_buf_size); return register_module_with_null_name(module_common, error_buf, @@ -1235,19 +1294,18 @@ wasm_runtime_dump_mem_consumption(WASMExecEnv *exec_env) &module_inst_mem_consps); wasm_get_module_mem_consumption(wasm_module, &module_mem_consps); if (wasm_module_inst->module->aux_stack_top_global_index != (uint32)-1) - max_aux_stack_used = wasm_module_inst->max_aux_stack_used; + max_aux_stack_used = wasm_module_inst->e->max_aux_stack_used; } #endif #if WASM_ENABLE_AOT != 0 if (module_inst_common->module_type == Wasm_Module_AoT) { AOTModuleInstance *aot_module_inst = (AOTModuleInstance *)module_inst_common; - AOTModule *aot_module = (AOTModule *)aot_module_inst->aot_module.ptr; + AOTModule *aot_module = (AOTModule *)aot_module_inst->module; module_common = (WASMModuleCommon *)aot_module; - if (aot_module_inst->memories.ptr) { - AOTMemoryInstance **memories = - (AOTMemoryInstance **)aot_module_inst->memories.ptr; - heap_handle = memories[0]->heap_handle.ptr; + if (aot_module_inst->memories) { + AOTMemoryInstance **memories = aot_module_inst->memories; + heap_handle = memories[0]->heap_handle; } aot_get_module_inst_mem_consumption(aot_module_inst, &module_inst_mem_consps); @@ -1982,99 +2040,130 @@ fail1: } bool -wasm_runtime_create_exec_env_singleton(WASMModuleInstanceCommon *module_inst) +wasm_runtime_create_exec_env_singleton( + WASMModuleInstanceCommon *module_inst_comm) { -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - return wasm_create_exec_env_singleton( - (WASMModuleInstance *)module_inst); -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - return aot_create_exec_env_singleton((AOTModuleInstance *)module_inst); -#endif - return false; + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMExecEnv *exec_env = NULL; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + if (module_inst->exec_env_singleton) { + return true; + } + + exec_env = wasm_exec_env_create(module_inst_comm, + module_inst->default_wasm_stack_size); + if (exec_env) + module_inst->exec_env_singleton = exec_env; + + return exec_env ? true : false; } WASMExecEnv * -wasm_runtime_get_exec_env_singleton(WASMModuleInstanceCommon *module_inst) +wasm_runtime_get_exec_env_singleton(WASMModuleInstanceCommon *module_inst_comm) { -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) { - if (!((WASMModuleInstance *)module_inst)->exec_env_singleton) { - wasm_create_exec_env_singleton((WASMModuleInstance *)module_inst); - } - return ((WASMModuleInstance *)module_inst)->exec_env_singleton; + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + if (!module_inst->exec_env_singleton) { + wasm_runtime_create_exec_env_singleton(module_inst_comm); } -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) { - if (!((AOTModuleInstance *)module_inst)->exec_env_singleton.ptr) { - aot_create_exec_env_singleton((AOTModuleInstance *)module_inst); - } - return (WASMExecEnv *)((AOTModuleInstance *)module_inst) - ->exec_env_singleton.ptr; - } -#endif - return NULL; + return module_inst->exec_env_singleton; } void -wasm_runtime_set_exception(WASMModuleInstanceCommon *module_inst, - const char *exception) +wasm_set_exception(WASMModuleInstance *module_inst, const char *exception) { -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) { - wasm_set_exception((WASMModuleInstance *)module_inst, exception); - return; - } -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) { - aot_set_exception((AOTModuleInstance *)module_inst, exception); - return; - } -#endif + if (exception) + snprintf(module_inst->cur_exception, sizeof(module_inst->cur_exception), + "Exception: %s", exception); + else + module_inst->cur_exception[0] = '\0'; +} + +/* clang-format off */ +static const char *exception_msgs[] = { + "unreachable", /* EXCE_UNREACHABLE */ + "allocate memory failed", /* EXCE_OUT_OF_MEMORY */ + "out of bounds memory access", /* EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS */ + "integer overflow", /* EXCE_INTEGER_OVERFLOW */ + "integer divide by zero", /* EXCE_INTEGER_DIVIDE_BY_ZERO */ + "invalid conversion to integer", /* EXCE_INVALID_CONVERSION_TO_INTEGER */ + "indirect call type mismatch", /* EXCE_INVALID_FUNCTION_TYPE_INDEX */ + "invalid function index", /* EXCE_INVALID_FUNCTION_INDEX */ + "undefined element", /* EXCE_UNDEFINED_ELEMENT */ + "uninitialized element", /* EXCE_UNINITIALIZED_ELEMENT */ + "failed to call unlinked import function", /* EXCE_CALL_UNLINKED_IMPORT_FUNC */ + "native stack overflow", /* EXCE_NATIVE_STACK_OVERFLOW */ + "unaligned atomic", /* EXCE_UNALIGNED_ATOMIC */ + "wasm auxiliary stack overflow", /* EXCE_AUX_STACK_OVERFLOW */ + "wasm auxiliary stack underflow", /* EXCE_AUX_STACK_UNDERFLOW */ + "out of bounds table access", /* EXCE_OUT_OF_BOUNDS_TABLE_ACCESS */ + "wasm operand stack overflow", /* EXCE_OPERAND_STACK_OVERFLOW */ + "", /* EXCE_ALREADY_THROWN */ +}; +/* clang-format on */ + +void +wasm_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id) +{ + if (id < EXCE_NUM) + wasm_set_exception(module_inst, exception_msgs[id]); + else + wasm_set_exception(module_inst, "unknown exception"); } const char * -wasm_runtime_get_exception(WASMModuleInstanceCommon *module_inst) +wasm_get_exception(WASMModuleInstance *module_inst) { -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) { - return wasm_get_exception((WASMModuleInstance *)module_inst); - } -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) { - return aot_get_exception((AOTModuleInstance *)module_inst); - } -#endif - return NULL; + if (module_inst->cur_exception[0] == '\0') + return NULL; + else + return module_inst->cur_exception; } void -wasm_runtime_clear_exception(WASMModuleInstanceCommon *module_inst) +wasm_runtime_set_exception(WASMModuleInstanceCommon *module_inst_comm, + const char *exception) { - wasm_runtime_set_exception(module_inst, NULL); + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + wasm_set_exception(module_inst, exception); +} + +const char * +wasm_runtime_get_exception(WASMModuleInstanceCommon *module_inst_comm) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + return wasm_get_exception(module_inst); } void -wasm_runtime_set_custom_data_internal(WASMModuleInstanceCommon *module_inst, - void *custom_data) +wasm_runtime_clear_exception(WASMModuleInstanceCommon *module_inst_comm) { -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) { - ((WASMModuleInstance *)module_inst)->custom_data = custom_data; - return; - } -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) { - ((AOTModuleInstance *)module_inst)->custom_data.ptr = custom_data; - return; - } -#endif + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + wasm_runtime_set_exception(module_inst_comm, NULL); +} + +void +wasm_runtime_set_custom_data_internal( + WASMModuleInstanceCommon *module_inst_comm, void *custom_data) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + module_inst->custom_data = custom_data; } void @@ -2089,17 +2178,13 @@ wasm_runtime_set_custom_data(WASMModuleInstanceCommon *module_inst, } void * -wasm_runtime_get_custom_data(WASMModuleInstanceCommon *module_inst) +wasm_runtime_get_custom_data(WASMModuleInstanceCommon *module_inst_comm) { -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - return ((WASMModuleInstance *)module_inst)->custom_data; -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - return ((AOTModuleInstance *)module_inst)->custom_data.ptr; -#endif - return NULL; + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + return module_inst->custom_data; } uint32 @@ -2171,155 +2256,6 @@ wasm_runtime_module_dup_data(WASMModuleInstanceCommon *module_inst, return 0; } -bool -wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst, - uint32 app_offset, uint32 size) -{ -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - return wasm_validate_app_addr((WASMModuleInstance *)module_inst, - app_offset, size); -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - return aot_validate_app_addr((AOTModuleInstance *)module_inst, - app_offset, size); -#endif - return false; -} - -bool -wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst, - uint32 app_str_offset) -{ - uint32 app_end_offset; - char *str, *str_end; - - if (!wasm_runtime_get_app_addr_range(module_inst, app_str_offset, NULL, - &app_end_offset)) - goto fail; - - str = wasm_runtime_addr_app_to_native(module_inst, app_str_offset); - str_end = str + (app_end_offset - app_str_offset); - while (str < str_end && *str != '\0') - str++; - if (str == str_end) - goto fail; - return true; - -fail: - wasm_runtime_set_exception(module_inst, "out of bounds memory access"); - return false; -} - -bool -wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst, - void *native_ptr, uint32 size) -{ -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - return wasm_validate_native_addr((WASMModuleInstance *)module_inst, - native_ptr, size); -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - return aot_validate_native_addr((AOTModuleInstance *)module_inst, - native_ptr, size); -#endif - return false; -} - -void * -wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst, - uint32 app_offset) -{ -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - return wasm_addr_app_to_native((WASMModuleInstance *)module_inst, - app_offset); -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - return aot_addr_app_to_native((AOTModuleInstance *)module_inst, - app_offset); -#endif - return NULL; -} - -uint32 -wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst, - void *native_ptr) -{ -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - return wasm_addr_native_to_app((WASMModuleInstance *)module_inst, - native_ptr); -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - return aot_addr_native_to_app((AOTModuleInstance *)module_inst, - native_ptr); -#endif - return 0; -} - -bool -wasm_runtime_get_app_addr_range(WASMModuleInstanceCommon *module_inst, - uint32 app_offset, uint32 *p_app_start_offset, - uint32 *p_app_end_offset) -{ -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - return wasm_get_app_addr_range((WASMModuleInstance *)module_inst, - app_offset, p_app_start_offset, - p_app_end_offset); -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - return aot_get_app_addr_range((AOTModuleInstance *)module_inst, - app_offset, p_app_start_offset, - p_app_end_offset); -#endif - return false; -} - -bool -wasm_runtime_get_native_addr_range(WASMModuleInstanceCommon *module_inst, - uint8 *native_ptr, - uint8 **p_native_start_addr, - uint8 **p_native_end_addr) -{ -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - return wasm_get_native_addr_range((WASMModuleInstance *)module_inst, - native_ptr, p_native_start_addr, - p_native_end_addr); -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - return aot_get_native_addr_range((AOTModuleInstance *)module_inst, - native_ptr, p_native_start_addr, - p_native_end_addr); -#endif - return false; -} - -bool -wasm_runtime_enlarge_memory(WASMModuleInstanceCommon *module, - uint32 inc_page_count) -{ -#if WASM_ENABLE_INTERP != 0 - if (module->module_type == Wasm_Module_Bytecode) - return wasm_enlarge_memory((WASMModuleInstance *)module, - inc_page_count); -#endif -#if WASM_ENABLE_AOT != 0 - if (module->module_type == Wasm_Module_AoT) - return aot_enlarge_memory((AOTModuleInstance *)module, inc_page_count); -#endif - return false; -} - #if WASM_ENABLE_LIBC_WASI != 0 static WASIArguments * @@ -2477,19 +2413,6 @@ wasm_runtime_init_wasi(WASMModuleInstanceCommon *module_inst, wasm_runtime_set_wasi_ctx(module_inst, wasi_ctx); -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode - && !((WASMModuleInstance *)module_inst)->default_memory) - return true; -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT - && !((AOTModuleInstance *)module_inst) - ->global_table_data.memory_instances[0] - .memory_data.ptr) - return true; -#endif - /* process argv[0], trip the path and suffix, only keep the program name */ if (!copy_string_array((const char **)argv, argc, &argv_buf, &argv_list, &argv_buf_size)) { @@ -2796,7 +2719,7 @@ wasm_runtime_is_wasi_mode(WASMModuleInstanceCommon *module_inst) #endif #if WASM_ENABLE_AOT != 0 if (module_inst->module_type == Wasm_Module_AoT - && ((AOTModule *)((AOTModuleInstance *)module_inst)->aot_module.ptr) + && ((AOTModule *)((AOTModuleInstance *)module_inst)->module) ->import_wasi_api) return true; #endif @@ -2832,7 +2755,7 @@ wasm_runtime_lookup_wasi_start_function(WASMModuleInstanceCommon *module_inst) if (module_inst->module_type == Wasm_Module_AoT) { AOTModuleInstance *aot_inst = (AOTModuleInstance *)module_inst; AOTFunctionInstance *export_funcs = - (AOTFunctionInstance *)aot_inst->export_funcs.ptr; + (AOTFunctionInstance *)aot_inst->export_functions; for (i = 0; i < aot_inst->export_func_count; i++) { if (!strcmp(export_funcs[i].func_name, "_start")) { AOTFuncType *func_type = export_funcs[i].u.func.func_type; @@ -2905,49 +2828,37 @@ wasm_runtime_destroy_wasi(WASMModuleInstanceCommon *module_inst) #endif WASIContext * -wasm_runtime_get_wasi_ctx(WASMModuleInstanceCommon *module_inst) +wasm_runtime_get_wasi_ctx(WASMModuleInstanceCommon *module_inst_comm) { -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - return ((WASMModuleInstance *)module_inst)->wasi_ctx; -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - return ((AOTModuleInstance *)module_inst)->wasi_ctx.ptr; -#endif - return NULL; + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + return module_inst->wasi_ctx; } void -wasm_runtime_set_wasi_ctx(WASMModuleInstanceCommon *module_inst, +wasm_runtime_set_wasi_ctx(WASMModuleInstanceCommon *module_inst_comm, WASIContext *wasi_ctx) { -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - ((WASMModuleInstance *)module_inst)->wasi_ctx = wasi_ctx; -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - ((AOTModuleInstance *)module_inst)->wasi_ctx.ptr = wasi_ctx; -#endif + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + module_inst->wasi_ctx = wasi_ctx; } #endif /* end of WASM_ENABLE_LIBC_WASI */ WASMModuleCommon * wasm_exec_env_get_module(WASMExecEnv *exec_env) { - WASMModuleInstanceCommon *module_inst = + WASMModuleInstanceCommon *module_inst_comm = wasm_runtime_get_module_inst(exec_env); -#if WASM_ENABLE_INTERP != 0 - if (module_inst->module_type == Wasm_Module_Bytecode) - return (WASMModuleCommon *)((WASMModuleInstance *)module_inst)->module; -#endif -#if WASM_ENABLE_AOT != 0 - if (module_inst->module_type == Wasm_Module_AoT) - return (WASMModuleCommon *)((AOTModuleInstance *)module_inst) - ->aot_module.ptr; -#endif - return NULL; + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + return (WASMModuleCommon *)module_inst->module; } #if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 @@ -4510,8 +4421,8 @@ interp_mark_all_externrefs(WASMModuleInstance *module_inst) WASMGlobalInstance *global; WASMTableInstance *table; - global = module_inst->globals; - for (i = 0; i < module_inst->global_count; i++, global++) { + global = module_inst->e->globals; + for (i = 0; i < module_inst->e->global_count; i++, global++) { if (global->type == VALUE_TYPE_EXTERNREF) { externref_idx = *(uint32 *)(global_data + global->data_offset); mark_externref(externref_idx); @@ -4519,14 +4430,23 @@ interp_mark_all_externrefs(WASMModuleInstance *module_inst) } for (i = 0; i < module_inst->table_count; i++) { + uint8 elem_type = 0; + uint32 init_size, max_size; + table = wasm_get_table_inst(module_inst, i); - if (table->elem_type == VALUE_TYPE_EXTERNREF) { - table_data = (uint32 *)table->base_addr; + (void)wasm_runtime_get_table_inst_elem_type( + (WASMModuleInstanceCommon *)module_inst, i, &elem_type, &init_size, + &max_size); + + if (elem_type == VALUE_TYPE_EXTERNREF) { + table_data = table->elems; for (j = 0; j < table->cur_size; j++) { externref_idx = table_data[j]; mark_externref(externref_idx); } } + (void)init_size; + (void)max_size; } } #endif @@ -4536,25 +4456,23 @@ static void aot_mark_all_externrefs(AOTModuleInstance *module_inst) { uint32 i = 0, j = 0; - const AOTModule *module = (AOTModule *)(module_inst->aot_module.ptr); + const AOTModule *module = (AOTModule *)module_inst->module; const AOTTable *table = module->tables; const AOTGlobal *global = module->globals; - const AOTTableInstance *table_inst = - (AOTTableInstance *)module_inst->tables.ptr; + const AOTTableInstance *table_inst; for (i = 0; i < module->global_count; i++, global++) { if (global->type == VALUE_TYPE_EXTERNREF) { - mark_externref(*(uint32 *)((uint8 *)module_inst->global_data.ptr - + global->data_offset)); + mark_externref( + *(uint32 *)(module_inst->global_data + global->data_offset)); } } - for (i = 0; i < module->table_count; - i++, table_inst = aot_next_tbl_inst(table_inst)) { - + for (i = 0; i < module->table_count; i++) { + table_inst = module_inst->tables[i]; if ((table + i)->elem_type == VALUE_TYPE_EXTERNREF) { while (j < table_inst->cur_size) { - mark_externref(table_inst->data[j++]); + mark_externref(table_inst->elems[j++]); } } } @@ -4707,6 +4625,82 @@ wasm_runtime_dump_call_stack_to_buf(wasm_exec_env_t exec_env, char *buf, } #endif /* end of WASM_ENABLE_DUMP_CALL_STACK */ +bool +wasm_runtime_get_table_elem_type(const WASMModuleCommon *module_comm, + uint32 table_idx, uint8 *out_elem_type, + uint32 *out_min_size, uint32 *out_max_size) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_comm->module_type == Wasm_Module_Bytecode) { + WASMModule *module = (WASMModule *)module_comm; + + if (table_idx < module->import_table_count) { + WASMTableImport *import_table = + &((module->import_tables + table_idx)->u.table); + *out_elem_type = import_table->elem_type; + *out_min_size = import_table->init_size; + *out_max_size = import_table->max_size; + } + else { + WASMTable *table = + module->tables + (table_idx - module->import_table_count); + *out_elem_type = table->elem_type; + *out_min_size = table->init_size; + *out_max_size = table->max_size; + } + return true; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_comm->module_type == Wasm_Module_AoT) { + AOTModule *module = (AOTModule *)module_comm; + + if (table_idx < module->import_table_count) { + AOTImportTable *import_table = module->import_tables + table_idx; + *out_elem_type = VALUE_TYPE_FUNCREF; + *out_min_size = import_table->table_init_size; + *out_max_size = import_table->table_max_size; + } + else { + AOTTable *table = + module->tables + (table_idx - module->import_table_count); + *out_elem_type = table->elem_type; + *out_min_size = table->table_init_size; + *out_max_size = table->table_max_size; + } + return true; + } +#endif + + return false; +} + +bool +wasm_runtime_get_table_inst_elem_type( + const WASMModuleInstanceCommon *module_inst_comm, uint32 table_idx, + uint8 *out_elem_type, uint32 *out_min_size, uint32 *out_max_size) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst_comm->module_type == Wasm_Module_Bytecode) { + WASMModuleInstance *module_inst = + (WASMModuleInstance *)module_inst_comm; + return wasm_runtime_get_table_elem_type( + (WASMModuleCommon *)module_inst->module, table_idx, out_elem_type, + out_min_size, out_max_size); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst_comm->module_type == Wasm_Module_AoT) { + AOTModuleInstance *module_inst = (AOTModuleInstance *)module_inst_comm; + return wasm_runtime_get_table_elem_type( + (WASMModuleCommon *)module_inst->module, table_idx, out_elem_type, + out_min_size, out_max_size); + } +#endif + return false; +} + bool wasm_runtime_get_export_func_type(const WASMModuleCommon *module_comm, const WASMExport *export, WASMType **out) @@ -4847,50 +4841,8 @@ wasm_runtime_get_export_table_type(const WASMModuleCommon *module_comm, uint8 *out_elem_type, uint32 *out_min_size, uint32 *out_max_size) { -#if WASM_ENABLE_INTERP != 0 - if (module_comm->module_type == Wasm_Module_Bytecode) { - WASMModule *module = (WASMModule *)module_comm; - - if (export->index < module->import_table_count) { - WASMTableImport *import_table = - &((module->import_tables + export->index)->u.table); - *out_elem_type = import_table->elem_type; - *out_min_size = import_table->init_size; - *out_max_size = import_table->max_size; - } - else { - WASMTable *table = - module->tables + (export->index - module->import_table_count); - *out_elem_type = table->elem_type; - *out_min_size = table->init_size; - *out_max_size = table->max_size; - } - return true; - } -#endif - -#if WASM_ENABLE_AOT != 0 - if (module_comm->module_type == Wasm_Module_AoT) { - AOTModule *module = (AOTModule *)module_comm; - - if (export->index < module->import_table_count) { - AOTImportTable *import_table = - module->import_tables + export->index; - *out_elem_type = VALUE_TYPE_FUNCREF; - *out_min_size = import_table->table_init_size; - *out_max_size = import_table->table_max_size; - } - else { - AOTTable *table = - module->tables + (export->index - module->import_table_count); - *out_elem_type = table->elem_type; - *out_min_size = table->table_init_size; - *out_max_size = table->table_max_size; - } - return true; - } -#endif - return false; + return wasm_runtime_get_table_elem_type( + module_comm, export->index, out_elem_type, out_min_size, out_max_size); } static inline bool @@ -5036,10 +4988,13 @@ wasm_runtime_invoke_c_api_native(WASMModuleInstanceCommon *module_inst, if (trap) { if (trap->message->data) { /* since trap->message->data does not end with '\0' */ - char trap_message[128] = { 0 }; - bh_memcpy_s(trap_message, 127, trap->message->data, - (trap->message->size < 127 ? (uint32)trap->message->size - : 127)); + char trap_message[108] = { 0 }; + uint32 max_size_to_copy = (uint32)sizeof(trap_message) - 1; + uint32 size_to_copy = (trap->message->size < max_size_to_copy) + ? (uint32)trap->message->size + : max_size_to_copy; + bh_memcpy_s(trap_message, (uint32)sizeof(trap_message), + trap->message->data, size_to_copy); wasm_runtime_set_exception(module_inst, trap_message); } else { diff --git a/core/iwasm/common/wasm_runtime_common.h b/core/iwasm/common/wasm_runtime_common.h index 87473397..e05ada14 100644 --- a/core/iwasm/common/wasm_runtime_common.h +++ b/core/iwasm/common/wasm_runtime_common.h @@ -852,17 +852,6 @@ wasm_runtime_dump_line_buf_impl(const char *line_buf, bool dump_or_print, WASMModuleCommon * wasm_exec_env_get_module(WASMExecEnv *exec_env); -/** - * Enlarge wasm memory data space. - * - * @param module the wasm module instance - * @param inc_page_count denote the page number to increase - * @return return true if enlarge successfully, false otherwise - */ -bool -wasm_runtime_enlarge_memory(WASMModuleInstanceCommon *module, - uint32 inc_page_count); - /* See wasm_export.h for description */ WASM_RUNTIME_API_EXTERN bool wasm_runtime_register_natives(const char *module_name, @@ -900,6 +889,16 @@ wasm_runtime_dump_module_inst_mem_consumption( void wasm_runtime_dump_exec_env_mem_consumption(const WASMExecEnv *exec_env); +bool +wasm_runtime_get_table_elem_type(const WASMModuleCommon *module_comm, + uint32 table_idx, uint8 *out_elem_type, + uint32 *out_min_size, uint32 *out_max_size); + +bool +wasm_runtime_get_table_inst_elem_type( + const WASMModuleInstanceCommon *module_inst_comm, uint32 table_idx, + uint8 *out_elem_type, uint32 *out_min_size, uint32 *out_max_size); + bool wasm_runtime_get_export_func_type(const WASMModuleCommon *module_comm, const WASMExport *export_, WASMType **out); diff --git a/core/iwasm/common/wasm_shared_memory.c b/core/iwasm/common/wasm_shared_memory.c index 395c677f..4ae88e89 100644 --- a/core/iwasm/common/wasm_shared_memory.c +++ b/core/iwasm/common/wasm_shared_memory.c @@ -314,44 +314,26 @@ uint32 wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address, uint64 expect, int64 timeout, bool wait64) { + WASMModuleInstance *module_inst = (WASMModuleInstance *)module; AtomicWaitInfo *wait_info; AtomicWaitNode *wait_node; bool check_ret, is_timeout; -#if WASM_ENABLE_INTERP != 0 - if (module->module_type == Wasm_Module_Bytecode) { - WASMModuleInstance *module_inst = (WASMModuleInstance *)module; - /* Currently we have only one memory instance */ - if (!module_inst->memories[0]->is_shared) { - wasm_runtime_set_exception(module, "expected shared memory"); - return -1; - } - if ((uint8 *)address < module_inst->memories[0]->memory_data - || (uint8 *)address + (wait64 ? 8 : 4) - > module_inst->memories[0]->memory_data_end) { - wasm_runtime_set_exception(module, "out of bounds memory access"); - return -1; - } + bh_assert(module->module_type == Wasm_Module_Bytecode + || module->module_type == Wasm_Module_AoT); + + /* Currently we have only one memory instance */ + if (!module_inst->memories[0]->is_shared) { + wasm_runtime_set_exception(module, "expected shared memory"); + return -1; } -#endif -#if WASM_ENABLE_AOT != 0 - if (module->module_type == Wasm_Module_AoT) { - AOTModuleInstance *aot_inst = (AOTModuleInstance *)module; - AOTMemoryInstance *aot_memory = - ((AOTMemoryInstance **)aot_inst->memories.ptr)[0]; - /* Currently we have only one memory instance */ - if (!aot_memory->is_shared) { - wasm_runtime_set_exception(module, "expected shared memory"); - return -1; - } - if ((uint8 *)address < (uint8 *)aot_memory->memory_data.ptr - || (uint8 *)address + (wait64 ? 8 : 4) - > (uint8 *)aot_memory->memory_data_end.ptr) { - wasm_runtime_set_exception(module, "out of bounds memory access"); - return -1; - } + + if ((uint8 *)address < module_inst->memories[0]->memory_data + || (uint8 *)address + (wait64 ? 8 : 4) + > module_inst->memories[0]->memory_data_end) { + wasm_runtime_set_exception(module, "out of bounds memory access"); + return -1; } -#endif /* acquire the wait info, create new one if not exists */ wait_info = acquire_wait_info(address, true); @@ -433,33 +415,18 @@ uint32 wasm_runtime_atomic_notify(WASMModuleInstanceCommon *module, void *address, uint32 count) { + WASMModuleInstance *module_inst = (WASMModuleInstance *)module; uint32 notify_result; AtomicWaitInfo *wait_info; -#if WASM_ENABLE_INTERP != 0 - if (module->module_type == Wasm_Module_Bytecode) { - WASMModuleInstance *module_inst = (WASMModuleInstance *)module; - if ((uint8 *)address < module_inst->memories[0]->memory_data - || (uint8 *)address + 4 - > module_inst->memories[0]->memory_data_end) { - wasm_runtime_set_exception(module, "out of bounds memory access"); - return -1; - } + bh_assert(module->module_type == Wasm_Module_Bytecode + || module->module_type == Wasm_Module_AoT); + + if ((uint8 *)address < module_inst->memories[0]->memory_data + || (uint8 *)address + 4 > module_inst->memories[0]->memory_data_end) { + wasm_runtime_set_exception(module, "out of bounds memory access"); + return -1; } -#endif -#if WASM_ENABLE_AOT != 0 - if (module->module_type == Wasm_Module_AoT) { - AOTModuleInstance *aot_inst = (AOTModuleInstance *)module; - AOTMemoryInstance *aot_memory = - ((AOTMemoryInstance **)aot_inst->memories.ptr)[0]; - if ((uint8 *)address < (uint8 *)aot_memory->memory_data.ptr - || (uint8 *)address + 4 - > (uint8 *)aot_memory->memory_data_end.ptr) { - wasm_runtime_set_exception(module, "out of bounds memory access"); - return -1; - } - } -#endif wait_info = acquire_wait_info(address, false); diff --git a/core/iwasm/compilation/aot.h b/core/iwasm/compilation/aot.h index a94bf646..8e8598c7 100644 --- a/core/iwasm/compilation/aot.h +++ b/core/iwasm/compilation/aot.h @@ -304,14 +304,22 @@ aot_set_last_error_v(const char *format, ...); #endif static inline uint32 -aot_get_tbl_data_slots(const AOTTable *tbl) +aot_get_imp_tbl_data_slots(const AOTImportTable *tbl, bool is_jit_mode) { +#if WASM_ENABLE_MULTI_MODULE != 0 + if (is_jit_mode) + return tbl->table_max_size; +#endif return tbl->possible_grow ? tbl->table_max_size : tbl->table_init_size; } static inline uint32 -aot_get_imp_tbl_data_slots(const AOTImportTable *tbl) +aot_get_tbl_data_slots(const AOTTable *tbl, bool is_jit_mode) { +#if WASM_ENABLE_MULTI_MODULE != 0 + if (is_jit_mode) + return tbl->table_max_size; +#endif return tbl->possible_grow ? tbl->table_max_size : tbl->table_init_size; } diff --git a/core/iwasm/compilation/aot_compiler.c b/core/iwasm/compilation/aot_compiler.c index 04425154..527c1a50 100644 --- a/core/iwasm/compilation/aot_compiler.c +++ b/core/iwasm/compilation/aot_compiler.c @@ -211,6 +211,7 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) case WASM_OP_BLOCK: case WASM_OP_LOOP: case WASM_OP_IF: + { value_type = *frame_ip++; if (value_type == VALUE_TYPE_I32 || value_type == VALUE_TYPE_I64 || value_type == VALUE_TYPE_F32 @@ -245,6 +246,25 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) param_count, param_types, result_count, result_types)) return false; break; + } + + case EXT_OP_BLOCK: + case EXT_OP_LOOP: + case EXT_OP_IF: + { + read_leb_uint32(frame_ip, frame_ip_end, type_index); + func_type = comp_ctx->comp_data->func_types[type_index]; + param_count = func_type->param_count; + param_types = func_type->types; + result_count = func_type->result_count; + result_types = func_type->types + param_count; + if (!aot_compile_op_block( + comp_ctx, func_ctx, &frame_ip, frame_ip_end, + (uint32)(LABEL_TYPE_BLOCK + opcode - EXT_OP_BLOCK), + param_count, param_types, result_count, result_types)) + return false; + break; + } case WASM_OP_ELSE: if (!aot_compile_op_else(comp_ctx, func_ctx, &frame_ip)) diff --git a/core/iwasm/compilation/aot_emit_const.c b/core/iwasm/compilation/aot_emit_const.c index 2508a36e..4b38aa96 100644 --- a/core/iwasm/compilation/aot_emit_const.c +++ b/core/iwasm/compilation/aot_emit_const.c @@ -4,7 +4,7 @@ */ #include "aot_emit_const.h" -#include "aot_intrinsic.h" +#include "../aot/aot_intrinsic.h" bool aot_compile_op_i32_const(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, diff --git a/core/iwasm/compilation/aot_emit_exception.c b/core/iwasm/compilation/aot_emit_exception.c index 3a16f7f8..d40ccc6a 100644 --- a/core/iwasm/compilation/aot_emit_exception.c +++ b/core/iwasm/compilation/aot_emit_exception.c @@ -4,6 +4,7 @@ */ #include "aot_emit_exception.h" +#include "../interpreter/wasm_runtime.h" #include "../aot/aot_runtime.h" bool @@ -57,7 +58,7 @@ aot_emit_exception(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } /* Create LLVM function with const function pointer */ if (!(func_const = - I64_CONST((uint64)(uintptr_t)aot_set_exception_with_id)) + I64_CONST((uint64)(uintptr_t)jit_set_exception_with_id)) || !(func = LLVMConstIntToPtr(func_const, func_ptr_type))) { aot_set_last_error("create LLVM value failed."); return false; diff --git a/core/iwasm/compilation/aot_emit_function.c b/core/iwasm/compilation/aot_emit_function.c index 1065b4c7..1bc4dbc2 100644 --- a/core/iwasm/compilation/aot_emit_function.c +++ b/core/iwasm/compilation/aot_emit_function.c @@ -162,7 +162,7 @@ call_aot_invoke_native_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } /* JIT mode, call the function directly */ - if (!(func = I64_CONST((uint64)(uintptr_t)aot_invoke_native)) + if (!(func = I64_CONST((uint64)(uintptr_t)llvm_jit_invoke_native)) || !(func = LLVMConstIntToPtr(func, func_ptr_type))) { aot_set_last_error("create LLVM value failed."); return false; @@ -407,7 +407,10 @@ call_aot_alloc_frame_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, param_types[1] = I32_TYPE; ret_type = INT8_TYPE; - GET_AOT_FUNCTION(aot_alloc_frame, 2); + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_alloc_frame, 2); + else + GET_AOT_FUNCTION(aot_alloc_frame, 2); param_values[0] = func_ctx->exec_env; param_values[1] = func_idx; @@ -461,7 +464,10 @@ call_aot_free_frame_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) param_types[0] = comp_ctx->exec_env_type; ret_type = INT8_TYPE; - GET_AOT_FUNCTION(aot_free_frame, 1); + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_free_frame, 1); + else + GET_AOT_FUNCTION(aot_free_frame, 1); param_values[0] = func_ctx->exec_env; @@ -557,7 +563,7 @@ check_app_addr_and_convert(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* JIT mode, call the function directly */ if (!(func = - I64_CONST((uint64)(uintptr_t)aot_check_app_addr_and_convert)) + I64_CONST((uint64)(uintptr_t)jit_check_app_addr_and_convert)) || !(func = LLVMConstIntToPtr(func, func_ptr_type))) { aot_set_last_error("create LLVM value failed."); return false; @@ -1047,7 +1053,7 @@ call_aot_call_indirect_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } /* JIT mode, call the function directly */ - if (!(func = I64_CONST((uint64)(uintptr_t)aot_call_indirect)) + if (!(func = I64_CONST((uint64)(uintptr_t)llvm_jit_call_indirect)) || !(func = LLVMConstIntToPtr(func, func_ptr_type))) { aot_set_last_error("create LLVM value failed."); return false; @@ -1265,7 +1271,7 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* load data as i32* */ if (!(offset = I32_CONST(get_tbl_inst_offset(comp_ctx, func_ctx, tbl_idx) - + offsetof(AOTTableInstance, data)))) { + + offsetof(AOTTableInstance, elems)))) { HANDLE_FAILURE("LLVMConstInt"); goto fail; } diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index 05e13b30..130f1830 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -718,7 +718,7 @@ aot_compile_op_memory_grow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) aot_set_last_error("llvm add pointer type failed."); return false; } - if (!(value = I64_CONST((uint64)(uintptr_t)aot_enlarge_memory)) + if (!(value = I64_CONST((uint64)(uintptr_t)wasm_enlarge_memory)) || !(func = LLVMConstIntToPtr(value, func_ptr_type))) { aot_set_last_error("create LLVM value failed."); return false; @@ -899,7 +899,10 @@ aot_compile_op_memory_init(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, param_types[4] = I32_TYPE; ret_type = INT8_TYPE; - GET_AOT_FUNCTION(aot_memory_init, 5); + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_memory_init, 5); + else + GET_AOT_FUNCTION(aot_memory_init, 5); /* Call function aot_memory_init() */ param_values[0] = func_ctx->aot_inst; @@ -955,7 +958,10 @@ aot_compile_op_data_drop(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, param_types[1] = I32_TYPE; ret_type = INT8_TYPE; - GET_AOT_FUNCTION(aot_data_drop, 2); + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_data_drop, 2); + else + GET_AOT_FUNCTION(aot_data_drop, 2); /* Call function aot_data_drop() */ param_values[0] = func_ctx->aot_inst; @@ -971,6 +977,14 @@ fail: return false; } +#if WASM_ENABLE_LAZY_JIT != 0 +static void * +jit_memmove(void *dest, const void *src, size_t n) +{ + return memmove(dest, src, n); +} +#endif + bool aot_compile_op_memory_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) { @@ -1025,7 +1039,7 @@ aot_compile_op_memory_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) return false; } #else - if (!(func = I64_CONST((uint64)(uintptr_t)aot_memmove)) + if (!(func = I64_CONST((uint64)(uintptr_t)jit_memmove)) || !(func = LLVMConstIntToPtr(func, func_ptr_type))) { aot_set_last_error("create LLVM value failed."); return false; @@ -1054,6 +1068,12 @@ fail: return false; } +static void * +jit_memset(void *s, int c, size_t n) +{ + return memset(s, c, n); +} + bool aot_compile_op_memory_fill(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) { @@ -1084,7 +1104,7 @@ aot_compile_op_memory_fill(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) } if (comp_ctx->is_jit_mode) { - if (!(func = I64_CONST((uint64)(uintptr_t)aot_memset)) + if (!(func = I64_CONST((uint64)(uintptr_t)jit_memset)) || !(func = LLVMConstIntToPtr(func, func_ptr_type))) { aot_set_last_error("create LLVM value failed."); return false; diff --git a/core/iwasm/compilation/aot_emit_table.c b/core/iwasm/compilation/aot_emit_table.c index 984ce8a0..9b18bd58 100644 --- a/core/iwasm/compilation/aot_emit_table.c +++ b/core/iwasm/compilation/aot_emit_table.c @@ -15,16 +15,25 @@ get_tbl_inst_offset(const AOTCompContext *comp_ctx, AOTImportTable *imp_tbls = comp_ctx->comp_data->import_tables; AOTTable *tbls = comp_ctx->comp_data->tables; - /* from the head of AOTModuleInstance */ - offset = - offsetof(AOTModuleInstance, global_table_data.bytes) - + (uint64)comp_ctx->comp_data->memory_count * sizeof(AOTMemoryInstance) - + comp_ctx->comp_data->global_data_size; + if (comp_ctx->is_jit_mode) { + offset = offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint64)comp_ctx->comp_data->memory_count + * sizeof(AOTMemoryInstance) + + comp_ctx->comp_data->global_data_size; + } + else { + offset = offsetof(AOTModuleInstance, global_table_data.bytes) + + (uint64)comp_ctx->comp_data->memory_count + * sizeof(AOTMemoryInstance) + + comp_ctx->comp_data->global_data_size; + } while (i < tbl_idx && i < comp_ctx->comp_data->import_table_count) { - offset += offsetof(AOTTableInstance, data); + offset += offsetof(AOTTableInstance, elems); /* avoid loading from current AOTTableInstance */ - offset += sizeof(uint32) * aot_get_imp_tbl_data_slots(imp_tbls + i); + offset += + sizeof(uint32) + * aot_get_imp_tbl_data_slots(imp_tbls + i, comp_ctx->is_jit_mode); ++i; } @@ -35,9 +44,10 @@ get_tbl_inst_offset(const AOTCompContext *comp_ctx, tbl_idx -= comp_ctx->comp_data->import_table_count; i -= comp_ctx->comp_data->import_table_count; while (i < tbl_idx && i < comp_ctx->comp_data->table_count) { - offset += offsetof(AOTTableInstance, data); + offset += offsetof(AOTTableInstance, elems); /* avoid loading from current AOTTableInstance */ - offset += sizeof(uint32) * aot_get_tbl_data_slots(tbls + i); + offset += sizeof(uint32) + * aot_get_tbl_data_slots(tbls + i, comp_ctx->is_jit_mode); ++i; } @@ -82,7 +92,10 @@ aot_compile_op_elem_drop(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, param_types[1] = I32_TYPE; ret_type = VOID_TYPE; - GET_AOT_FUNCTION(aot_drop_table_seg, 2); + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_drop_table_seg, 2); + else + GET_AOT_FUNCTION(aot_drop_table_seg, 2); param_values[0] = func_ctx->aot_inst; if (!(param_values[1] = I32_CONST(tbl_seg_idx))) { @@ -176,7 +189,7 @@ aot_compile_op_table_get(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* load data as i32* */ if (!(offset = I32_CONST(get_tbl_inst_offset(comp_ctx, func_ctx, tbl_idx) - + offsetof(AOTTableInstance, data)))) { + + offsetof(AOTTableInstance, elems)))) { HANDLE_FAILURE("LLVMConstInt"); goto fail; } @@ -230,7 +243,7 @@ aot_compile_op_table_set(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* load data as i32* */ if (!(offset = I32_CONST(get_tbl_inst_offset(comp_ctx, func_ctx, tbl_idx) - + offsetof(AOTTableInstance, data)))) { + + offsetof(AOTTableInstance, elems)))) { HANDLE_FAILURE("LLVMConstInt"); goto fail; } @@ -282,7 +295,10 @@ aot_compile_op_table_init(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, param_types[5] = I32_TYPE; ret_type = VOID_TYPE; - GET_AOT_FUNCTION(aot_table_init, 6); + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_table_init, 6); + else + GET_AOT_FUNCTION(aot_table_init, 6); param_values[0] = func_ctx->aot_inst; @@ -330,7 +346,10 @@ aot_compile_op_table_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, param_types[5] = I32_TYPE; ret_type = VOID_TYPE; - GET_AOT_FUNCTION(aot_table_copy, 6); + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_table_copy, 6); + else + GET_AOT_FUNCTION(aot_table_copy, 6); param_values[0] = func_ctx->aot_inst; @@ -414,7 +433,10 @@ aot_compile_op_table_grow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, param_types[3] = I32_TYPE; ret_type = I32_TYPE; - GET_AOT_FUNCTION(aot_table_grow, 4); + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_table_grow, 4); + else + GET_AOT_FUNCTION(aot_table_grow, 4); param_values[0] = func_ctx->aot_inst; @@ -455,7 +477,10 @@ aot_compile_op_table_fill(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, param_types[4] = I32_TYPE; ret_type = VOID_TYPE; - GET_AOT_FUNCTION(aot_table_fill, 5); + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_table_fill, 5); + else + GET_AOT_FUNCTION(aot_table_fill, 5); param_values[0] = func_ctx->aot_inst; diff --git a/core/iwasm/compilation/aot_emit_variable.c b/core/iwasm/compilation/aot_emit_variable.c index 4510a4c3..590638ce 100644 --- a/core/iwasm/compilation/aot_emit_variable.c +++ b/core/iwasm/compilation/aot_emit_variable.c @@ -114,14 +114,23 @@ compile_global(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, { AOTCompData *comp_data = comp_ctx->comp_data; uint32 import_global_count = comp_data->import_global_count; - uint32 global_base_offset = - offsetof(AOTModuleInstance, global_table_data.bytes) - + sizeof(AOTMemoryInstance) * comp_ctx->comp_data->memory_count; + uint32 global_base_offset; uint32 global_offset; uint8 global_type; LLVMValueRef offset, global_ptr, global, res; LLVMTypeRef ptr_type = NULL; + if (comp_ctx->is_jit_mode) { + global_base_offset = + offsetof(WASMModuleInstance, global_table_data.bytes) + + sizeof(WASMMemoryInstance) * comp_ctx->comp_data->memory_count; + } + else { + global_base_offset = + offsetof(AOTModuleInstance, global_table_data.bytes) + + sizeof(AOTMemoryInstance) * comp_ctx->comp_data->memory_count; + } + bh_assert(global_idx < import_global_count + comp_data->global_count); if (global_idx < import_global_count) { diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index 5f052bf2..aa65fcc5 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -264,7 +264,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return false; } /* memories[0]->memory_data */ - offset = I32_CONST(offsetof(AOTMemoryInstance, memory_data.ptr)); + offset = I32_CONST(offsetof(AOTMemoryInstance, memory_data)); if (!(func_ctx->mem_info[0].mem_base_addr = LLVMBuildInBoundsGEP2( comp_ctx->builder, INT8_TYPE, shared_mem_addr, &offset, 1, "mem_base_addr_offset"))) { @@ -292,15 +292,24 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, else #endif { - offset = I32_CONST(offsetof(AOTModuleInstance, global_table_data) - + offsetof(AOTMemoryInstance, memory_data.ptr)); + uint32 offset_of_global_table_data; + + if (comp_ctx->is_jit_mode) + offset_of_global_table_data = + offsetof(WASMModuleInstance, global_table_data); + else + offset_of_global_table_data = + offsetof(AOTModuleInstance, global_table_data); + + offset = I32_CONST(offset_of_global_table_data + + offsetof(AOTMemoryInstance, memory_data)); if (!(func_ctx->mem_info[0].mem_base_addr = LLVMBuildInBoundsGEP2( comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst, &offset, 1, "mem_base_addr_offset"))) { aot_set_last_error("llvm build in bounds gep failed"); return false; } - offset = I32_CONST(offsetof(AOTModuleInstance, global_table_data) + offset = I32_CONST(offset_of_global_table_data + offsetof(AOTMemoryInstance, cur_page_count)); if (!(func_ctx->mem_info[0].mem_cur_page_count_addr = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, @@ -309,7 +318,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, aot_set_last_error("llvm build in bounds gep failed"); return false; } - offset = I32_CONST(offsetof(AOTModuleInstance, global_table_data) + offset = I32_CONST(offset_of_global_table_data + offsetof(AOTMemoryInstance, memory_data_size)); if (!(func_ctx->mem_info[0].mem_data_size_addr = LLVMBuildInBoundsGEP2( comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst, &offset, 1, @@ -379,7 +388,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* Load memory bound check constants */ offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_1byte) - - offsetof(AOTMemoryInstance, memory_data.ptr)); + - offsetof(AOTMemoryInstance, memory_data)); if (!(func_ctx->mem_info[0].mem_bound_check_1byte = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, mem_info_base, &offset, 1, "bound_check_1byte_offset"))) { @@ -405,7 +414,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_2bytes) - - offsetof(AOTMemoryInstance, memory_data.ptr)); + - offsetof(AOTMemoryInstance, memory_data)); if (!(func_ctx->mem_info[0].mem_bound_check_2bytes = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, mem_info_base, &offset, 1, "bound_check_2bytes_offset"))) { @@ -431,7 +440,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_4bytes) - - offsetof(AOTMemoryInstance, memory_data.ptr)); + - offsetof(AOTMemoryInstance, memory_data)); if (!(func_ctx->mem_info[0].mem_bound_check_4bytes = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, mem_info_base, &offset, 1, "bound_check_4bytes_offset"))) { @@ -457,7 +466,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_8bytes) - - offsetof(AOTMemoryInstance, memory_data.ptr)); + - offsetof(AOTMemoryInstance, memory_data)); if (!(func_ctx->mem_info[0].mem_bound_check_8bytes = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, mem_info_base, &offset, 1, "bound_check_8bytes_offset"))) { @@ -483,7 +492,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_16bytes) - - offsetof(AOTMemoryInstance, memory_data.ptr)); + - offsetof(AOTMemoryInstance, memory_data)); if (!(func_ctx->mem_info[0].mem_bound_check_16bytes = LLVMBuildInBoundsGEP2( comp_ctx->builder, INT8_TYPE, mem_info_base, &offset, 1, "bound_check_16bytes_offset"))) { @@ -533,7 +542,7 @@ create_func_type_indexes(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) LLVMValueRef offset, func_type_indexes_ptr; LLVMTypeRef int32_ptr_type; - offset = I32_CONST(offsetof(AOTModuleInstance, func_type_indexes.ptr)); + offset = I32_CONST(offsetof(AOTModuleInstance, func_type_indexes)); func_type_indexes_ptr = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst, &offset, 1, "func_type_indexes_ptr"); diff --git a/core/iwasm/compilation/aot_llvm_extra.cpp b/core/iwasm/compilation/aot_llvm_extra.cpp index ed9fabd9..f0c205a8 100644 --- a/core/iwasm/compilation/aot_llvm_extra.cpp +++ b/core/iwasm/compilation/aot_llvm_extra.cpp @@ -338,8 +338,8 @@ aot_lookup_orcjit_func(LLVMOrcLLJITRef orc_lazyjit, void *module_inst, LLVMErrorRef error; LLVMOrcJITTargetAddress func_addr = 0; AOTModuleInstance *aot_inst = (AOTModuleInstance *)module_inst; - AOTModule *aot_module = (AOTModule *)aot_inst->aot_module.ptr; - void **func_ptrs = (void **)aot_inst->func_ptrs.ptr; + AOTModule *aot_module = (AOTModule *)aot_inst->module; + void **func_ptrs = aot_inst->func_ptrs; /** * No need to lock the func_ptr[func_idx] here as it is basic diff --git a/core/iwasm/fast-jit/fe/jit_emit_control.c b/core/iwasm/fast-jit/fe/jit_emit_control.c index eafce674..95e356c3 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_control.c +++ b/core/iwasm/fast-jit/fe/jit_emit_control.c @@ -1200,7 +1200,7 @@ jit_compile_op_return(JitCompContext *cc, uint8 **p_frame_ip) bool jit_compile_op_unreachable(JitCompContext *cc, uint8 **p_frame_ip) { - if (!jit_emit_exception(cc, JIT_EXCE_UNREACHABLE, JIT_OP_JMP, 0, NULL)) + if (!jit_emit_exception(cc, EXCE_UNREACHABLE, JIT_OP_JMP, 0, NULL)) return false; return handle_next_reachable_block(cc, p_frame_ip); diff --git a/core/iwasm/fast-jit/fe/jit_emit_conversion.c b/core/iwasm/fast-jit/fe/jit_emit_conversion.c index e753f929..8308a3ca 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_conversion.c +++ b/core/iwasm/fast-jit/fe/jit_emit_conversion.c @@ -201,19 +201,19 @@ jit_compile_check_value_range(JitCompContext *cc, JitReg value, JitReg min_fp, goto fail; GEN_INSN(CMP, cc->cmp_reg, nan_ret, NEW_CONST(I32, 1)); - if (!jit_emit_exception(cc, JIT_EXCE_INVALID_CONVERSION_TO_INTEGER, - JIT_OP_BEQ, cc->cmp_reg, NULL)) + if (!jit_emit_exception(cc, EXCE_INVALID_CONVERSION_TO_INTEGER, JIT_OP_BEQ, + cc->cmp_reg, NULL)) goto fail; /* If value is out of integer range, throw exception */ GEN_INSN(CMP, cc->cmp_reg, min_fp, value); - if (!jit_emit_exception(cc, JIT_EXCE_INTEGER_OVERFLOW, JIT_OP_BGES, - cc->cmp_reg, NULL)) + if (!jit_emit_exception(cc, EXCE_INTEGER_OVERFLOW, JIT_OP_BGES, cc->cmp_reg, + NULL)) goto fail; GEN_INSN(CMP, cc->cmp_reg, value, max_fp); - if (!jit_emit_exception(cc, JIT_EXCE_INTEGER_OVERFLOW, JIT_OP_BGES, - cc->cmp_reg, NULL)) + if (!jit_emit_exception(cc, EXCE_INTEGER_OVERFLOW, JIT_OP_BGES, cc->cmp_reg, + NULL)) goto fail; return true; diff --git a/core/iwasm/fast-jit/fe/jit_emit_exception.c b/core/iwasm/fast-jit/fe/jit_emit_exception.c index 5e227685..2addb5cd 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_exception.c +++ b/core/iwasm/fast-jit/fe/jit_emit_exception.c @@ -14,7 +14,7 @@ jit_emit_exception(JitCompContext *cc, int32 exception_id, uint8 jit_opcode, JitIncomingInsn *incoming_insn; JitReg else_label; - bh_assert(exception_id < JIT_EXCE_NUM); + bh_assert(exception_id < EXCE_NUM); if (jit_opcode >= JIT_OP_BEQ && jit_opcode <= JIT_OP_BLEU) { bh_assert(cond_br_if == cc->cmp_reg); diff --git a/core/iwasm/fast-jit/fe/jit_emit_function.c b/core/iwasm/fast-jit/fe/jit_emit_function.c index 257615f0..7f83231c 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_function.c +++ b/core/iwasm/fast-jit/fe/jit_emit_function.c @@ -9,10 +9,6 @@ #include "../jit_codegen.h" #include "../../interpreter/wasm_runtime.h" -extern bool -jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, - WASMInterpFrame *prev_frame); - /* Prepare parameters for the function to call */ static bool pre_call(JitCompContext *cc, const WASMType *func_type) @@ -187,8 +183,8 @@ jit_compile_op_call(JitCompContext *cc, uint32 func_idx, bool tail_call) arg_regs[1] = NEW_CONST(I32, func_idx); arg_regs[2] = cc->fp_reg; - if (!jit_emit_callnative(cc, jit_invoke_native, native_ret, arg_regs, - 3)) { + if (!jit_emit_callnative(cc, fast_jit_invoke_native, native_ret, + arg_regs, 3)) { return false; } /* Convert bool to uint32 */ @@ -196,7 +192,7 @@ jit_compile_op_call(JitCompContext *cc, uint32 func_idx, bool tail_call) /* Check whether there is exception thrown */ GEN_INSN(CMP, cc->cmp_reg, native_ret, NEW_CONST(I32, 0)); - if (!jit_emit_exception(cc, JIT_EXCE_ALREADY_THROWN, JIT_OP_BEQ, + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BEQ, cc->cmp_reg, NULL)) { return false; } @@ -355,7 +351,8 @@ jit_compile_op_call_indirect(JitCompContext *cc, uint32 type_idx, arg_regs[4] = NEW_CONST(I32, func_type->param_cell_num); arg_regs[5] = argv; - if (!jit_emit_callnative(cc, jit_call_indirect, native_ret, arg_regs, 6)) { + if (!jit_emit_callnative(cc, fast_jit_call_indirect, native_ret, arg_regs, + 6)) { return false; } /* Convert bool to uint32 */ @@ -363,8 +360,8 @@ jit_compile_op_call_indirect(JitCompContext *cc, uint32 type_idx, /* Check whether there is exception thrown */ GEN_INSN(CMP, cc->cmp_reg, native_ret, NEW_CONST(I32, 0)); - if (!jit_emit_exception(cc, JIT_EXCE_ALREADY_THROWN, JIT_OP_BEQ, - cc->cmp_reg, NULL)) { + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BEQ, cc->cmp_reg, + NULL)) { return false; } diff --git a/core/iwasm/fast-jit/fe/jit_emit_memory.c b/core/iwasm/fast-jit/fe/jit_emit_memory.c index 6daf5c0d..19687039 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_memory.c +++ b/core/iwasm/fast-jit/fe/jit_emit_memory.c @@ -78,8 +78,8 @@ check_and_seek_on_64bit_platform(JitCompContext *cc, JitReg addr, JitReg offset, #ifndef OS_ENABLE_HW_BOUND_CHECK /* if (offset1 > memory_boundary) goto EXCEPTION */ GEN_INSN(CMP, cc->cmp_reg, offset1, memory_boundary); - if (!jit_emit_exception(cc, JIT_EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, - JIT_OP_BGTU, cc->cmp_reg, NULL)) { + if (!jit_emit_exception(cc, EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, JIT_OP_BGTU, + cc->cmp_reg, NULL)) { goto fail; } #endif @@ -103,16 +103,16 @@ check_and_seek_on_32bit_platform(JitCompContext *cc, JitReg addr, JitReg offset, /* if (offset1 < addr) goto EXCEPTION */ GEN_INSN(CMP, cc->cmp_reg, offset1, addr); - if (!jit_emit_exception(cc, JIT_EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, - JIT_OP_BLTU, cc->cmp_reg, NULL)) { + if (!jit_emit_exception(cc, EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, JIT_OP_BLTU, + cc->cmp_reg, NULL)) { goto fail; } #ifndef OS_ENABLE_HW_BOUND_CHECK /* if (offset1 > memory_boundary) goto EXCEPTION */ GEN_INSN(CMP, cc->cmp_reg, offset1, memory_boundary); - if (!jit_emit_exception(cc, JIT_EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, - JIT_OP_BGTU, cc->cmp_reg, NULL)) { + if (!jit_emit_exception(cc, EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, JIT_OP_BGTU, + cc->cmp_reg, NULL)) { goto fail; } #endif @@ -144,7 +144,7 @@ check_and_seek(JitCompContext *cc, JitReg addr, uint32 offset, uint32 bytes) GEN_INSN(LDI32, cur_mem_page_count, memory_inst, NEW_CONST(I32, offsetof(WASMMemoryInstance, cur_page_count))); GEN_INSN(CMP, cc->cmp_reg, cur_mem_page_count, NEW_CONST(I32, 0)); - if (!jit_emit_exception(cc, JIT_EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, + if (!jit_emit_exception(cc, EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, JIT_OP_BEQ, cc->cmp_reg, NULL)) { goto fail; } @@ -604,8 +604,8 @@ jit_compile_op_memory_init(JitCompContext *cc, uint32 mem_idx, uint32 seg_idx) goto fail; GEN_INSN(CMP, cc->cmp_reg, res, NEW_CONST(I32, 0)); - if (!jit_emit_exception(cc, JIT_EXCE_ALREADY_THROWN, JIT_OP_BLTS, - cc->cmp_reg, NULL)) + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BLTS, cc->cmp_reg, + NULL)) goto fail; return true; @@ -687,8 +687,8 @@ jit_compile_op_memory_copy(JitCompContext *cc, uint32 src_mem_idx, goto fail; GEN_INSN(CMP, cc->cmp_reg, res, NEW_CONST(I32, 0)); - if (!jit_emit_exception(cc, JIT_EXCE_ALREADY_THROWN, JIT_OP_BLTS, - cc->cmp_reg, NULL)) + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BLTS, cc->cmp_reg, + NULL)) goto fail; return true; @@ -741,8 +741,8 @@ jit_compile_op_memory_fill(JitCompContext *cc, uint32 mem_idx) goto fail; GEN_INSN(CMP, cc->cmp_reg, res, NEW_CONST(I32, 0)); - if (!jit_emit_exception(cc, JIT_EXCE_ALREADY_THROWN, JIT_OP_BLTS, - cc->cmp_reg, NULL)) + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BLTS, cc->cmp_reg, + NULL)) goto fail; return true; diff --git a/core/iwasm/fast-jit/fe/jit_emit_numberic.c b/core/iwasm/fast-jit/fe/jit_emit_numberic.c index 3b2f21af..04c8f780 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_numberic.c +++ b/core/iwasm/fast-jit/fe/jit_emit_numberic.c @@ -665,7 +665,7 @@ compile_int_div(JitCompContext *cc, IntArithmetic arith_op, bool is_i32, case 0: { /* Directly throw exception if divided by zero */ - if (!(jit_emit_exception(cc, JIT_EXCE_INTEGER_DIVIDE_BY_ZERO, + if (!(jit_emit_exception(cc, EXCE_INTEGER_DIVIDE_BY_ZERO, JIT_OP_JMP, 0, NULL))) goto fail; @@ -699,7 +699,7 @@ compile_int_div(JitCompContext *cc, IntArithmetic arith_op, bool is_i32, /* Throw integer overflow exception if left is INT32_MIN or INT64_MIN */ - if (!(jit_emit_exception(cc, JIT_EXCE_INTEGER_OVERFLOW, + if (!(jit_emit_exception(cc, EXCE_INTEGER_OVERFLOW, JIT_OP_BEQ, cc->cmp_reg, NULL))) goto fail; @@ -739,8 +739,8 @@ compile_int_div(JitCompContext *cc, IntArithmetic arith_op, bool is_i32, GEN_INSN(CMP, cc->cmp_reg, right, is_i32 ? NEW_CONST(I32, 0) : NEW_CONST(I64, 0)); /* Throw integer divided by zero exception if right is zero */ - if (!(jit_emit_exception(cc, JIT_EXCE_INTEGER_DIVIDE_BY_ZERO, - JIT_OP_BEQ, cc->cmp_reg, NULL))) + if (!(jit_emit_exception(cc, EXCE_INTEGER_DIVIDE_BY_ZERO, JIT_OP_BEQ, + cc->cmp_reg, NULL))) goto fail; switch (arith_op) { @@ -760,8 +760,8 @@ compile_int_div(JitCompContext *cc, IntArithmetic arith_op, bool is_i32, GEN_INSN(CMP, cc->cmp_reg, cmp1, NEW_CONST(I32, 1)); /* Throw integer overflow exception if left is INT32_MIN or INT64_MIN, and right is -1 */ - if (!(jit_emit_exception(cc, JIT_EXCE_INTEGER_OVERFLOW, - JIT_OP_BEQ, cc->cmp_reg, NULL))) + if (!(jit_emit_exception(cc, EXCE_INTEGER_OVERFLOW, JIT_OP_BEQ, + cc->cmp_reg, NULL))) goto fail; /* Build default div and rem */ diff --git a/core/iwasm/fast-jit/fe/jit_emit_table.c b/core/iwasm/fast-jit/fe/jit_emit_table.c index 57017113..0b24a663 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_table.c +++ b/core/iwasm/fast-jit/fe/jit_emit_table.c @@ -37,8 +37,8 @@ jit_compile_op_table_get(JitCompContext *cc, uint32 tbl_idx) /* if (elem_idx >= tbl_sz) goto exception; */ tbl_sz = get_table_cur_size_reg(cc->jit_frame, tbl_idx); GEN_INSN(CMP, cc->cmp_reg, elem_idx, tbl_sz); - if (!jit_emit_exception(cc, JIT_EXCE_OUT_OF_BOUNDS_TABLE_ACCESS, - JIT_OP_BGEU, cc->cmp_reg, NULL)) + if (!jit_emit_exception(cc, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS, JIT_OP_BGEU, + cc->cmp_reg, NULL)) goto fail; elem_idx_long = jit_cc_new_reg_I64(cc); @@ -68,8 +68,8 @@ jit_compile_op_table_set(JitCompContext *cc, uint32 tbl_idx) /* if (elem_idx >= tbl_sz) goto exception; */ tbl_sz = get_table_cur_size_reg(cc->jit_frame, tbl_idx); GEN_INSN(CMP, cc->cmp_reg, elem_idx, tbl_sz); - if (!jit_emit_exception(cc, JIT_EXCE_OUT_OF_BOUNDS_TABLE_ACCESS, - JIT_OP_BGEU, cc->cmp_reg, NULL)) + if (!jit_emit_exception(cc, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS, JIT_OP_BGEU, + cc->cmp_reg, NULL)) goto fail; elem_idx_long = jit_cc_new_reg_I64(cc); @@ -105,7 +105,7 @@ wasm_init_table(WASMModuleInstance *inst, uint32 tbl_idx, uint32 elem_idx, if (src > elem_len || elem_len - src < len) goto out_of_bounds; - bh_memcpy_s((uint8 *)(tbl) + offsetof(WASMTableInstance, base_addr) + bh_memcpy_s((uint8 *)tbl + offsetof(WASMTableInstance, elems) + dst * sizeof(uint32), (uint32)((tbl_sz - dst) * sizeof(uint32)), elem->func_indexes + src, (uint32)(len * sizeof(uint32))); @@ -140,8 +140,8 @@ jit_compile_op_table_init(JitCompContext *cc, uint32 tbl_idx, goto fail; GEN_INSN(CMP, cc->cmp_reg, res, NEW_CONST(I32, 0)); - if (!jit_emit_exception(cc, JIT_EXCE_ALREADY_THROWN, JIT_OP_BLTS, - cc->cmp_reg, NULL)) + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BLTS, cc->cmp_reg, + NULL)) goto fail; return true; @@ -167,10 +167,10 @@ wasm_copy_table(WASMModuleInstance *inst, uint32 src_tbl_idx, if (dst_offset > dst_tbl_sz || dst_tbl_sz - dst_offset < len) goto out_of_bounds; - bh_memmove_s((uint8 *)(dst_tbl) + offsetof(WASMTableInstance, base_addr) + bh_memmove_s((uint8 *)dst_tbl + offsetof(WASMTableInstance, elems) + dst_offset * sizeof(uint32), (uint32)((dst_tbl_sz - dst_offset) * sizeof(uint32)), - (uint8 *)(src_tbl) + offsetof(WASMTableInstance, base_addr) + (uint8 *)src_tbl + offsetof(WASMTableInstance, elems) + src_offset * sizeof(uint32), (uint32)(len * sizeof(uint32))); @@ -204,8 +204,8 @@ jit_compile_op_table_copy(JitCompContext *cc, uint32 src_tbl_idx, goto fail; GEN_INSN(CMP, cc->cmp_reg, res, NEW_CONST(I32, 0)); - if (!jit_emit_exception(cc, JIT_EXCE_ALREADY_THROWN, JIT_OP_BLTS, - cc->cmp_reg, NULL)) + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BLTS, cc->cmp_reg, + NULL)) goto fail; return true; @@ -276,7 +276,7 @@ wasm_fill_table(WASMModuleInstance *inst, uint32 tbl_idx, uint32 dst, goto out_of_bounds; for (; len != 0; dst++, len--) { - ((uint32 *)(tbl->base_addr))[dst] = val; + tbl->elems[dst] = val; } return 0; @@ -307,8 +307,8 @@ jit_compile_op_table_fill(JitCompContext *cc, uint32 tbl_idx) goto fail; GEN_INSN(CMP, cc->cmp_reg, res, NEW_CONST(I32, 0)); - if (!jit_emit_exception(cc, JIT_EXCE_ALREADY_THROWN, JIT_OP_BLTS, - cc->cmp_reg, NULL)) + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BLTS, cc->cmp_reg, + NULL)) goto fail; return true; diff --git a/core/iwasm/fast-jit/fe/jit_emit_variable.c b/core/iwasm/fast-jit/fe/jit_emit_variable.c index e018d220..ba039d38 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_variable.c +++ b/core/iwasm/fast-jit/fe/jit_emit_variable.c @@ -277,11 +277,11 @@ jit_compile_op_set_global(JitCompContext *cc, uint32 global_idx, JitReg aux_stack_bottom = get_aux_stack_bottom_reg(cc->jit_frame); GEN_INSN(CMP, cc->cmp_reg, value, aux_stack_bound); - if (!(jit_emit_exception(cc, JIT_EXCE_AUX_STACK_OVERFLOW, + if (!(jit_emit_exception(cc, EXCE_AUX_STACK_OVERFLOW, JIT_OP_BLEU, cc->cmp_reg, NULL))) goto fail; GEN_INSN(CMP, cc->cmp_reg, value, aux_stack_bottom); - if (!(jit_emit_exception(cc, JIT_EXCE_AUX_STACK_UNDERFLOW, + if (!(jit_emit_exception(cc, EXCE_AUX_STACK_UNDERFLOW, JIT_OP_BGTU, cc->cmp_reg, NULL))) goto fail; } diff --git a/core/iwasm/fast-jit/jit_frontend.c b/core/iwasm/fast-jit/jit_frontend.c index 90e20da3..dc65123f 100644 --- a/core/iwasm/fast-jit/jit_frontend.c +++ b/core/iwasm/fast-jit/jit_frontend.c @@ -18,31 +18,9 @@ #include "fe/jit_emit_variable.h" #include "../interpreter/wasm_interp.h" #include "../interpreter/wasm_opcode.h" +#include "../interpreter/wasm_runtime.h" #include "../common/wasm_exec_env.h" -/* clang-format off */ -static const char *jit_exception_msgs[] = { - "unreachable", /* JIT_EXCE_UNREACHABLE */ - "allocate memory failed", /* JIT_EXCE_OUT_OF_MEMORY */ - "out of bounds memory access", /* JIT_EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS */ - "integer overflow", /* JIT_EXCE_INTEGER_OVERFLOW */ - "integer divide by zero", /* JIT_EXCE_INTEGER_DIVIDE_BY_ZERO */ - "invalid conversion to integer", /* JIT_EXCE_INVALID_CONVERSION_TO_INTEGER */ - "indirect call type mismatch", /* JIT_EXCE_INVALID_FUNCTION_TYPE_INDEX */ - "invalid function index", /* JIT_EXCE_INVALID_FUNCTION_INDEX */ - "undefined element", /* JIT_EXCE_UNDEFINED_ELEMENT */ - "uninitialized element", /* JIT_EXCE_UNINITIALIZED_ELEMENT */ - "failed to call unlinked import function", /* JIT_EXCE_CALL_UNLINKED_IMPORT_FUNC */ - "native stack overflow", /* JIT_EXCE_NATIVE_STACK_OVERFLOW */ - "unaligned atomic", /* JIT_EXCE_UNALIGNED_ATOMIC */ - "wasm auxiliary stack overflow", /* JIT_EXCE_AUX_STACK_OVERFLOW */ - "wasm auxiliary stack underflow", /* JIT_EXCE_AUX_STACK_UNDERFLOW */ - "out of bounds table access", /* JIT_EXCE_OUT_OF_BOUNDS_TABLE_ACCESS */ - "wasm operand stack overflow", /* JIT_EXCE_OPERAND_STACK_OVERFLOW */ - "", /* JIT_EXCE_ALREADY_THROWN */ -}; -/* clang-format on */ - JitReg get_module_inst_reg(JitFrame *frame) { @@ -378,7 +356,7 @@ get_table_data_reg(JitFrame *frame, uint32 tbl_idx) frame->table_regs[tbl_idx].table_data = cc->table_regs[tbl_idx].table_data; GEN_INSN(ADD, frame->table_regs[tbl_idx].table_data, table_reg, - NEW_CONST(I64, offsetof(WASMTableInstance, base_addr))); + NEW_CONST(I64, offsetof(WASMTableInstance, elems))); } return frame->table_regs[tbl_idx].table_data; } @@ -586,15 +564,6 @@ gen_commit_sp_ip(JitFrame *frame) #endif } -static void -jit_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id) -{ - if (id < JIT_EXCE_NUM) - wasm_set_exception(module_inst, jit_exception_msgs[id]); - else - wasm_set_exception(module_inst, "unknown exception"); -} - static bool create_fixed_virtual_regs(JitCompContext *cc) { @@ -681,7 +650,7 @@ form_and_translate_func(JitCompContext *cc) jit_basic_block_append_insn(jit_cc_entry_basic_block(cc), insn); /* Patch INSNs jumping to exception basic blocks. */ - for (i = 0; i < JIT_EXCE_NUM; i++) { + for (i = 0; i < EXCE_NUM; i++) { incoming_insn = cc->incoming_insns_for_exec_bbs[i]; if (incoming_insn) { if (!(cc->exce_basic_blocks[i] = jit_cc_new_basic_block(cc, 0))) { @@ -703,7 +672,7 @@ form_and_translate_func(JitCompContext *cc) incoming_insn = incoming_insn_next; } cc->cur_basic_block = cc->exce_basic_blocks[i]; - if (i != JIT_EXCE_ALREADY_THROWN) { + if (i != EXCE_ALREADY_THROWN) { JitReg module_inst_reg = jit_cc_new_reg_ptr(cc); GEN_INSN(LDPTR, module_inst_reg, cc->exec_env_reg, NEW_CONST(I32, offsetof(WASMExecEnv, module_inst))); @@ -850,7 +819,7 @@ init_func_translation(JitCompContext *cc) GEN_INSN(ADD, frame_boundary, top, NEW_CONST(PTR, frame_size + outs_size)); /* if frame_boundary > top_boundary, throw stack overflow exception */ GEN_INSN(CMP, cc->cmp_reg, frame_boundary, top_boundary); - if (!jit_emit_exception(cc, JIT_EXCE_OPERAND_STACK_OVERFLOW, JIT_OP_BGTU, + if (!jit_emit_exception(cc, EXCE_OPERAND_STACK_OVERFLOW, JIT_OP_BGTU, cc->cmp_reg, NULL)) { return NULL; } diff --git a/core/iwasm/fast-jit/jit_frontend.h b/core/iwasm/fast-jit/jit_frontend.h index eeecf48f..725d930a 100644 --- a/core/iwasm/fast-jit/jit_frontend.h +++ b/core/iwasm/fast-jit/jit_frontend.h @@ -104,28 +104,6 @@ typedef enum FloatArithmetic { FLOAT_MAX, } FloatArithmetic; -typedef enum JitExceptionID { - JIT_EXCE_UNREACHABLE = 0, - JIT_EXCE_OUT_OF_MEMORY, - JIT_EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, - JIT_EXCE_INTEGER_OVERFLOW, - JIT_EXCE_INTEGER_DIVIDE_BY_ZERO, - JIT_EXCE_INVALID_CONVERSION_TO_INTEGER, - JIT_EXCE_INVALID_FUNCTION_TYPE_INDEX, - JIT_EXCE_INVALID_FUNCTION_INDEX, - JIT_EXCE_UNDEFINED_ELEMENT, - JIT_EXCE_UNINITIALIZED_ELEMENT, - JIT_EXCE_CALL_UNLINKED_IMPORT_FUNC, - JIT_EXCE_NATIVE_STACK_OVERFLOW, - JIT_EXCE_UNALIGNED_ATOMIC, - JIT_EXCE_AUX_STACK_OVERFLOW, - JIT_EXCE_AUX_STACK_UNDERFLOW, - JIT_EXCE_OUT_OF_BOUNDS_TABLE_ACCESS, - JIT_EXCE_OPERAND_STACK_OVERFLOW, - JIT_EXCE_ALREADY_THROWN, - JIT_EXCE_NUM, -} JitExceptionID; - /** * Translate instructions in a function. The translated block must * end with a branch instruction whose targets are offsets relating to diff --git a/core/iwasm/fast-jit/jit_ir.c b/core/iwasm/fast-jit/jit_ir.c index 6efc2eff..9fc8d5cc 100644 --- a/core/iwasm/fast-jit/jit_ir.c +++ b/core/iwasm/fast-jit/jit_ir.c @@ -382,6 +382,7 @@ jit_cc_init(JitCompContext *cc, unsigned htab_size) blocks respectively. */ if (!(entry_block = jit_cc_new_basic_block(cc, 0))) goto fail; + if (!(exit_block = jit_cc_new_basic_block(cc, 0))) { jit_basic_block_delete(entry_block); goto fail; @@ -395,11 +396,11 @@ jit_cc_init(JitCompContext *cc, unsigned htab_size) && jit_reg_no(cc->exit_label) == 1); if (!(cc->exce_basic_blocks = - jit_calloc(sizeof(JitBasicBlock *) * JIT_EXCE_NUM))) + jit_calloc(sizeof(JitBasicBlock *) * EXCE_NUM))) goto fail; if (!(cc->incoming_insns_for_exec_bbs = - jit_calloc(sizeof(JitIncomingInsnList) * JIT_EXCE_NUM))) + jit_calloc(sizeof(JitIncomingInsnList) * EXCE_NUM))) goto fail; cc->hreg_info = jit_codegen_get_hreg_info(); @@ -465,7 +466,7 @@ jit_cc_destroy(JitCompContext *cc) jit_free(cc->exce_basic_blocks); if (cc->incoming_insns_for_exec_bbs) { - for (i = 0; i < JIT_EXCE_NUM; i++) { + for (i = 0; i < EXCE_NUM; i++) { incoming_insn = cc->incoming_insns_for_exec_bbs[i]; while (incoming_insn) { incoming_insn_next = incoming_insn->next; diff --git a/core/iwasm/include/wasm_c_api.h b/core/iwasm/include/wasm_c_api.h index 1c9d6a18..84832364 100644 --- a/core/iwasm/include/wasm_c_api.h +++ b/core/iwasm/include/wasm_c_api.h @@ -461,7 +461,7 @@ WASM_API_EXTERN bool wasm_module_validate(wasm_store_t*, const wasm_byte_vec_t* WASM_API_EXTERN void wasm_module_imports(const wasm_module_t*, own wasm_importtype_vec_t* out); WASM_API_EXTERN void wasm_module_exports(const wasm_module_t*, own wasm_exporttype_vec_t* out); -WASM_API_EXTERN void wasm_module_serialize(const wasm_module_t*, own wasm_byte_vec_t* out); +WASM_API_EXTERN void wasm_module_serialize(wasm_module_t*, own wasm_byte_vec_t* out); WASM_API_EXTERN own wasm_module_t* wasm_module_deserialize(wasm_store_t*, const wasm_byte_vec_t*); diff --git a/core/iwasm/interpreter/wasm.h b/core/iwasm/interpreter/wasm.h index 44b3973d..ad0972b7 100644 --- a/core/iwasm/interpreter/wasm.h +++ b/core/iwasm/interpreter/wasm.h @@ -263,6 +263,9 @@ struct WASMFunction { #if WASM_ENABLE_FAST_JIT != 0 void *fast_jit_jitted_code; #endif +#if WASM_ENABLE_JIT != 0 + void *llvm_jit_func_ptr; +#endif }; struct WASMGlobal { @@ -363,6 +366,11 @@ typedef struct WASMCustomSection { } WASMCustomSection; #endif +#if WASM_ENABLE_JIT != 0 +struct AOTCompData; +struct AOTCompContext; +#endif + struct WASMModule { /* Module type, for module loaded from WASM bytecode binary, this field is Wasm_Module_Bytecode; @@ -407,6 +415,9 @@ struct WASMModule { WASMDataSeg **data_segments; uint32 start_function; + /* total global variable size */ + uint32 global_data_size; + /* the index of auxiliary __data_end global, -1 means unexported */ uint32 aux_data_end_global_index; @@ -493,6 +504,12 @@ struct WASMModule { /* point to JITed functions */ void **fast_jit_func_ptrs; #endif + +#if WASM_ENABLE_JIT != 0 + struct AOTCompData *comp_data; + struct AOTCompContext *comp_ctx; + void **func_ptrs; +#endif }; typedef struct BlockType { diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index b7edcf7c..30c40baf 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -641,28 +641,28 @@ trunc_f64_to_int(WASMModuleInstance *module, uint32 *frame_sp, float64 src_min, CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = (uint32)(*(uint8 *)maddr); \ *(uint8 *)maddr = (uint8)(readv op sval); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ else if (opcode == WASM_OP_ATOMIC_RMW_I32_##OP_NAME##16_U) { \ CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = (uint32)LOAD_U16(maddr); \ STORE_U16(maddr, (uint16)(readv op sval)); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ else { \ CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = LOAD_I32(maddr); \ STORE_U32(maddr, readv op sval); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ PUSH_I32(readv); \ break; \ @@ -681,39 +681,39 @@ trunc_f64_to_int(WASMModuleInstance *module, uint32 *frame_sp, float64 src_min, CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = (uint64)(*(uint8 *)maddr); \ *(uint8 *)maddr = (uint8)(readv op sval); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##16_U) { \ CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = (uint64)LOAD_U16(maddr); \ STORE_U16(maddr, (uint16)(readv op sval)); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##32_U) { \ CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = (uint64)LOAD_U32(maddr); \ STORE_U32(maddr, (uint32)(readv op sval)); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ else { \ uint64 op_result; \ CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = (uint64)LOAD_I64(maddr); \ op_result = readv op sval; \ STORE_I64(maddr, op_result); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ PUSH_I64(readv); \ break; \ @@ -850,7 +850,7 @@ wasm_interp_call_func_native(WASMModuleInstance *module_inst, wasm_exec_env_set_cur_frame(exec_env, frame); - cur_func_index = (uint32)(cur_func - module_inst->functions); + cur_func_index = (uint32)(cur_func - module_inst->e->functions); bh_assert(cur_func_index < module_inst->module->import_function_count); native_func_pointer = module_inst->import_func_ptrs[cur_func_index]; @@ -904,12 +904,12 @@ wasm_interp_call_func_native(WASMModuleInstance *module_inst, #if WASM_ENABLE_FAST_JIT != 0 bool -jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, - WASMInterpFrame *prev_frame) +fast_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, + WASMInterpFrame *prev_frame) { WASMModuleInstance *module_inst = (WASMModuleInstance *)exec_env->module_inst; - WASMFunctionInstance *cur_func = module_inst->functions + func_idx; + WASMFunctionInstance *cur_func = module_inst->e->functions + func_idx; wasm_interp_call_func_native(module_inst, exec_env, cur_func, prev_frame); return wasm_get_exception(module_inst) ? false : true; @@ -1081,7 +1081,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, WASMFunctionInstance *cur_func, WASMInterpFrame *prev_frame) { - WASMMemoryInstance *memory = module->default_memory; + WASMMemoryInstance *memory = wasm_get_default_memory(module); uint8 *global_data = module->global_data; #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ @@ -1091,7 +1091,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, memory ? num_bytes_per_page * memory->cur_page_count : 0; #endif WASMType **wasm_types = module->module->types; - WASMGlobalInstance *globals = module->globals, *global; + WASMGlobalInstance *globals = module->e->globals, *global; uint8 opcode_IMPDEP = WASM_OP_IMPDEP; WASMInterpFrame *frame = NULL; /* Points to this special opcode so as to jump to the @@ -1355,13 +1355,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #endif read_leb_uint32(frame_ip, frame_ip_end, fidx); #if WASM_ENABLE_MULTI_MODULE != 0 - if (fidx >= module->function_count) { + if (fidx >= module->e->function_count) { wasm_set_exception(module, "unknown function"); goto got_exception; } #endif - cur_func = module->functions + fidx; + cur_func = module->e->functions + fidx; goto call_func_from_interp; } @@ -1373,12 +1373,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #endif read_leb_uint32(frame_ip, frame_ip_end, fidx); #if WASM_ENABLE_MULTI_MODULE != 0 - if (fidx >= module->function_count) { + if (fidx >= module->e->function_count) { wasm_set_exception(module, "unknown function"); goto got_exception; } #endif - cur_func = module->functions + fidx; + cur_func = module->e->functions + fidx; goto call_func_from_return_call; } @@ -1420,8 +1420,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; } - fidx = ((uint32 *)tbl_inst->base_addr)[val]; - if (fidx == (uint32)-1) { + fidx = tbl_inst->elems[val]; + if (fidx == NULL_REF) { wasm_set_exception(module, "uninitialized element"); goto got_exception; } @@ -1431,13 +1431,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, * another module. In that case, we don't validate * the elem value while loading */ - if (fidx >= module->function_count) { + if (fidx >= module->e->function_count) { wasm_set_exception(module, "unknown function"); goto got_exception; } /* always call module own functions */ - cur_func = module->functions + fidx; + cur_func = module->e->functions + fidx; if (cur_func->is_import_func) cur_func_type = cur_func->u.func_import->func_type; @@ -1531,7 +1531,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; } - PUSH_I32(((uint32 *)tbl_inst->base_addr)[elem_idx]); + PUSH_I32(tbl_inst->elems[elem_idx]); HANDLE_OP_END(); } @@ -1552,7 +1552,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; } - ((uint32 *)(tbl_inst->base_addr))[elem_idx] = elem_val; + tbl_inst->elems[elem_idx] = elem_val; HANDLE_OP_END(); } @@ -1700,7 +1700,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_GET_GLOBAL) { read_leb_uint32(frame_ip, frame_ip_end, global_idx); - bh_assert(global_idx < module->global_count); + bh_assert(global_idx < module->e->global_count); global = globals + global_idx; global_addr = get_global_addr(global_data, global); PUSH_I32(*(uint32 *)global_addr); @@ -1710,7 +1710,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_GET_GLOBAL_64) { read_leb_uint32(frame_ip, frame_ip_end, global_idx); - bh_assert(global_idx < module->global_count); + bh_assert(global_idx < module->e->global_count); global = globals + global_idx; global_addr = get_global_addr(global_data, global); PUSH_I64(GET_I64_FROM_ADDR((uint32 *)global_addr)); @@ -1720,7 +1720,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_SET_GLOBAL) { read_leb_uint32(frame_ip, frame_ip_end, global_idx); - bh_assert(global_idx < module->global_count); + bh_assert(global_idx < module->e->global_count); global = globals + global_idx; global_addr = get_global_addr(global_data, global); *(int32 *)global_addr = POP_I32(); @@ -1732,7 +1732,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, uint32 aux_stack_top; read_leb_uint32(frame_ip, frame_ip_end, global_idx); - bh_assert(global_idx < module->global_count); + bh_assert(global_idx < module->e->global_count); global = globals + global_idx; global_addr = get_global_addr(global_data, global); aux_stack_top = *(uint32 *)(frame_sp - 1); @@ -1751,8 +1751,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (module->module->aux_stack_top_global_index != (uint32)-1) { uint32 aux_stack_used = module->module->aux_stack_bottom - *(uint32 *)global_addr; - if (aux_stack_used > module->max_aux_stack_used) - module->max_aux_stack_used = aux_stack_used; + if (aux_stack_used > module->e->max_aux_stack_used) + module->e->max_aux_stack_used = aux_stack_used; } #endif HANDLE_OP_END(); @@ -1761,7 +1761,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_SET_GLOBAL_64) { read_leb_uint32(frame_ip, frame_ip_end, global_idx); - bh_assert(global_idx < module->global_count); + bh_assert(global_idx < module->e->global_count); global = globals + global_idx; global_addr = get_global_addr(global_data, global); PUT_I64_TO_ADDR((uint32 *)global_addr, POP_I64()); @@ -2039,8 +2039,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, else { /* success, return previous page count */ PUSH_I32(prev_page_count); - /* update memory instance ptr and memory size */ - memory = module->default_memory; + /* update memory size, no need to update memory ptr as + it isn't changed in wasm_enlarge_memory */ #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ || WASM_ENABLE_BULK_MEMORY != 0 @@ -3195,8 +3195,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, } bh_memcpy_s( - (uint8 *)(tbl_inst) - + offsetof(WASMTableInstance, base_addr) + (uint8 *)tbl_inst + + offsetof(WASMTableInstance, elems) + d * sizeof(uint32), (uint32)((tbl_inst->cur_size - d) * sizeof(uint32)), module->module->table_segments[elem_idx] @@ -3246,16 +3246,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, /* if s >= d, copy from front to back */ /* if s < d, copy from back to front */ /* merge all together */ - bh_memmove_s( - (uint8 *)(dst_tbl_inst) - + offsetof(WASMTableInstance, base_addr) - + d * sizeof(uint32), - (uint32)((dst_tbl_inst->cur_size - d) - * sizeof(uint32)), - (uint8 *)(src_tbl_inst) - + offsetof(WASMTableInstance, base_addr) - + s * sizeof(uint32), - (uint32)(n * sizeof(uint32))); + bh_memmove_s((uint8 *)dst_tbl_inst + + offsetof(WASMTableInstance, elems) + + d * sizeof(uint32), + (uint32)((dst_tbl_inst->cur_size - d) + * sizeof(uint32)), + (uint8 *)src_tbl_inst + + offsetof(WASMTableInstance, elems) + + s * sizeof(uint32), + (uint32)(n * sizeof(uint32))); break; } case WASM_OP_TABLE_GROW: @@ -3319,7 +3318,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, } for (; n != 0; i++, n--) { - ((uint32 *)(tbl_inst->base_addr))[i] = fill_val; + tbl_inst->elems[i] = fill_val; } break; @@ -3420,23 +3419,23 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (opcode == WASM_OP_ATOMIC_I32_LOAD8_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint32)(*(uint8 *)maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_I32_LOAD16_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint32)LOAD_U16(maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = LOAD_I32(maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } PUSH_I32(readv); @@ -3455,30 +3454,30 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (opcode == WASM_OP_ATOMIC_I64_LOAD8_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)(*(uint8 *)maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_I64_LOAD16_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)LOAD_U16(maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_I64_LOAD32_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)LOAD_U32(maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = LOAD_I64(maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } PUSH_I64(readv); @@ -3497,23 +3496,23 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (opcode == WASM_OP_ATOMIC_I32_STORE8) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); *(uint8 *)maddr = (uint8)sval; - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_I32_STORE16) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); STORE_U16(maddr, (uint16)sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); STORE_U32(maddr, frame_sp[1]); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } break; } @@ -3531,31 +3530,31 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (opcode == WASM_OP_ATOMIC_I64_STORE8) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); *(uint8 *)maddr = (uint8)sval; - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_I64_STORE16) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); STORE_U16(maddr, (uint16)sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_I64_STORE32) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); STORE_U32(maddr, (uint32)sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); PUT_I64_TO_ADDR((uint32 *)maddr, GET_I64_FROM_ADDR(frame_sp + 1)); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } break; } @@ -3575,32 +3574,32 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, CHECK_ATOMIC_MEMORY_ACCESS(); expect = (uint8)expect; - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint32)(*(uint8 *)maddr); if (readv == expect) *(uint8 *)maddr = (uint8)(sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); expect = (uint16)expect; - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint32)LOAD_U16(maddr); if (readv == expect) STORE_U16(maddr, (uint16)(sval)); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = LOAD_I32(maddr); if (readv == expect) STORE_U32(maddr, sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } PUSH_I32(readv); break; @@ -3621,44 +3620,44 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, CHECK_ATOMIC_MEMORY_ACCESS(); expect = (uint8)expect; - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)(*(uint8 *)maddr); if (readv == expect) *(uint8 *)maddr = (uint8)(sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); expect = (uint16)expect; - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)LOAD_U16(maddr); if (readv == expect) STORE_U16(maddr, (uint16)(sval)); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); expect = (uint32)expect; - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)LOAD_U32(maddr); if (readv == expect) STORE_U32(maddr, (uint32)(sval)); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); CHECK_ATOMIC_MEMORY_ACCESS(); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)LOAD_I64(maddr); if (readv == expect) { STORE_I64(maddr, sval); } - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } PUSH_I64(readv); break; @@ -3794,8 +3793,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, cur_func = frame->function; UPDATE_ALL_FROM_FRAME(); - /* update memory instance ptr and memory size */ - memory = module->default_memory; + /* update memory size, no need to update memory ptr as + it isn't changed in wasm_enlarge_memory */ #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ || WASM_ENABLE_BULK_MEMORY != 0 @@ -3905,6 +3904,191 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #endif } +#if WASM_ENABLE_FAST_JIT != 0 +static void +fast_jit_call_func_bytecode(WASMExecEnv *exec_env, + WASMFunctionInstance *function, + WASMInterpFrame *frame) +{ + JitGlobals *jit_globals = jit_compiler_get_jit_globals(); + JitInterpSwitchInfo info; + WASMType *func_type = function->u.func->func_type; + uint8 type = func_type->result_count + ? func_type->types[func_type->param_count] + : VALUE_TYPE_VOID; + +#if WASM_ENABLE_REF_TYPES != 0 + if (type == VALUE_TYPE_EXTERNREF || type == VALUE_TYPE_FUNCREF) + type = VALUE_TYPE_I32; +#endif + + info.out.ret.last_return_type = type; + info.frame = frame; + frame->jitted_return_addr = + (uint8 *)jit_globals->return_to_interp_from_jitted; + jit_interp_switch_to_jitted(exec_env, &info, + function->u.func->fast_jit_jitted_code); + if (func_type->result_count) { + switch (type) { + case VALUE_TYPE_I32: + *(frame->sp - function->ret_cell_num) = info.out.ret.ival[0]; + break; + case VALUE_TYPE_I64: + *(frame->sp - function->ret_cell_num) = info.out.ret.ival[0]; + *(frame->sp - function->ret_cell_num + 1) = + info.out.ret.ival[1]; + break; + case VALUE_TYPE_F32: + *(frame->sp - function->ret_cell_num) = info.out.ret.fval[0]; + break; + case VALUE_TYPE_F64: + *(frame->sp - function->ret_cell_num) = info.out.ret.fval[0]; + *(frame->sp - function->ret_cell_num + 1) = + info.out.ret.fval[1]; + break; + default: + bh_assert(0); + break; + } + } +} +#endif + +#if WASM_ENABLE_JIT != 0 +static bool +clear_wasi_proc_exit_exception(WASMModuleInstance *module_inst) +{ +#if WASM_ENABLE_LIBC_WASI != 0 + const char *exception = wasm_get_exception(module_inst); + if (exception && !strcmp(exception, "Exception: wasi proc exit")) { + /* The "wasi proc exit" exception is thrown by native lib to + let wasm app exit, which is a normal behavior, we clear + the exception here. */ + wasm_set_exception(module_inst, NULL); + return true; + } + return false; +#else + return false; +#endif +} + +static bool +llvm_jit_call_func_bytecode(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, + WASMFunctionInstance *function, uint32 argc, + uint32 argv[]) +{ + WASMType *func_type = function->u.func->func_type; + uint32 result_count = func_type->result_count; + uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0; + bool ret; + +#if (WASM_ENABLE_DUMP_CALL_STACK != 0) || (WASM_ENABLE_PERF_PROFILING != 0) + if (!llvm_jit_alloc_frame(exec_env, function - module_inst->e->functions)) { + wasm_set_exception(module_inst, "wasm operand stack overflow"); + } +#endif + + if (ext_ret_count > 0) { + uint32 cell_num = 0, i; + uint8 *ext_ret_types = func_type->types + func_type->param_count + 1; + uint32 argv1_buf[32], *argv1 = argv1_buf, *ext_rets = NULL; + uint32 *argv_ret = argv; + uint32 ext_ret_cell = wasm_get_cell_num(ext_ret_types, ext_ret_count); + uint64 size; + + /* Allocate memory all arguments */ + size = + sizeof(uint32) * (uint64)argc /* original arguments */ + + sizeof(void *) + * (uint64)ext_ret_count /* extra result values' addr */ + + sizeof(uint32) * (uint64)ext_ret_cell; /* extra result values */ + if (size > sizeof(argv1_buf)) { + if (size > UINT32_MAX + || !(argv1 = wasm_runtime_malloc((uint32)size))) { + wasm_set_exception(module_inst, "allocate memory failed"); + return false; + } + } + + /* Copy original arguments */ + bh_memcpy_s(argv1, (uint32)size, argv, sizeof(uint32) * argc); + + /* Get the extra result value's address */ + ext_rets = + argv1 + argc + sizeof(void *) / sizeof(uint32) * ext_ret_count; + + /* Append each extra result value's address to original arguments */ + for (i = 0; i < ext_ret_count; i++) { + *(uintptr_t *)(argv1 + argc + sizeof(void *) / sizeof(uint32) * i) = + (uintptr_t)(ext_rets + cell_num); + cell_num += wasm_value_type_cell_num(ext_ret_types[i]); + } + + ret = wasm_runtime_invoke_native( + exec_env, function->u.func->llvm_jit_func_ptr, func_type, NULL, + NULL, argv1, argc, argv); + + if (!ret || wasm_get_exception(module_inst)) { + if (clear_wasi_proc_exit_exception(module_inst)) + ret = true; + else + ret = false; + } + + if (!ret) { + if (argv1 != argv1_buf) + wasm_runtime_free(argv1); + return ret; + } + + /* Get extra result values */ + switch (func_type->types[func_type->param_count]) { + case VALUE_TYPE_I32: + case VALUE_TYPE_F32: +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: +#endif + argv_ret++; + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + argv_ret += 2; + break; +#if WASM_ENABLE_SIMD != 0 + case VALUE_TYPE_V128: + argv_ret += 4; + break; +#endif + default: + bh_assert(0); + break; + } + + ext_rets = + argv1 + argc + sizeof(void *) / sizeof(uint32) * ext_ret_count; + bh_memcpy_s(argv_ret, sizeof(uint32) * cell_num, ext_rets, + sizeof(uint32) * cell_num); + + if (argv1 != argv1_buf) + wasm_runtime_free(argv1); + return true; + } + else { + ret = wasm_runtime_invoke_native( + exec_env, function->u.func->llvm_jit_func_ptr, func_type, NULL, + NULL, argv, argc, argv); + + if (clear_wasi_proc_exit_exception(module_inst)) + ret = true; + + return ret && !wasm_get_exception(module_inst) ? true : false; + } +} +#endif + void wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env, WASMFunctionInstance *function, uint32 argc, @@ -3912,14 +4096,14 @@ wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env, { WASMRuntimeFrame *prev_frame = wasm_exec_env_get_cur_frame(exec_env); WASMInterpFrame *frame, *outs_area; - /* Allocate sufficient cells for all kinds of return values. */ unsigned all_cell_num = - function->ret_cell_num > 2 ? function->ret_cell_num : 2, - i; + function->ret_cell_num > 2 ? function->ret_cell_num : 2; /* This frame won't be used by JITed code, so only allocate interp frame here. */ unsigned frame_size = wasm_interp_interp_frame_size(all_cell_num); + unsigned i; + bool copy_argv_from_frame = true; if (argc < function->param_cell_num) { char buf[128]; @@ -3969,62 +4153,26 @@ wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env, } } else { -#if WASM_ENABLE_FAST_JIT == 0 - wasm_interp_call_func_bytecode(module_inst, exec_env, function, frame); +#if WASM_ENABLE_JIT != 0 + llvm_jit_call_func_bytecode(module_inst, exec_env, function, argc, + argv); + /* For llvm jit, the results have been stored in argv, + no need to copy them from stack frame again */ + copy_argv_from_frame = false; +#elif WASM_ENABLE_FAST_JIT != 0 + fast_jit_call_func_bytecode(exec_env, function, frame); #else - JitGlobals *jit_globals = jit_compiler_get_jit_globals(); - JitInterpSwitchInfo info; - WASMType *func_type = function->u.func->func_type; - uint8 type = func_type->result_count - ? func_type->types[func_type->param_count] - : VALUE_TYPE_VOID; - -#if WASM_ENABLE_REF_TYPES != 0 - if (type == VALUE_TYPE_EXTERNREF || type == VALUE_TYPE_FUNCREF) - type = VALUE_TYPE_I32; + wasm_interp_call_func_bytecode(module_inst, exec_env, function, frame); #endif - - info.out.ret.last_return_type = type; - info.frame = frame; - frame->jitted_return_addr = - (uint8 *)jit_globals->return_to_interp_from_jitted; - jit_interp_switch_to_jitted(exec_env, &info, - function->u.func->fast_jit_jitted_code); - if (func_type->result_count) { - switch (type) { - case VALUE_TYPE_I32: - *(frame->sp - function->ret_cell_num) = - info.out.ret.ival[0]; - break; - case VALUE_TYPE_I64: - *(frame->sp - function->ret_cell_num) = - info.out.ret.ival[0]; - *(frame->sp - function->ret_cell_num + 1) = - info.out.ret.ival[1]; - break; - case VALUE_TYPE_F32: - *(frame->sp - function->ret_cell_num) = - info.out.ret.fval[0]; - break; - case VALUE_TYPE_F64: - *(frame->sp - function->ret_cell_num) = - info.out.ret.fval[0]; - *(frame->sp - function->ret_cell_num + 1) = - info.out.ret.fval[1]; - break; - default: - bh_assert(0); - break; - } - } (void)wasm_interp_call_func_bytecode; -#endif } /* Output the return value to the caller */ if (!wasm_get_exception(module_inst)) { - for (i = 0; i < function->ret_cell_num; i++) { - argv[i] = *(frame->sp + i - function->ret_cell_num); + if (copy_argv_from_frame) { + for (i = 0; i < function->ret_cell_num; i++) { + argv[i] = *(frame->sp + i - function->ret_cell_num); + } } } else { diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 2f9c654a..d2440d18 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -443,28 +443,28 @@ LOAD_PTR(void *addr) CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(1); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = (uint32)(*(uint8 *)maddr); \ *(uint8 *)maddr = (uint8)(readv op sval); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ else if (opcode == WASM_OP_ATOMIC_RMW_I32_##OP_NAME##16_U) { \ CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(2); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = (uint32)LOAD_U16(maddr); \ STORE_U16(maddr, (uint16)(readv op sval)); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ else { \ CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(4); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = LOAD_I32(maddr); \ STORE_U32(maddr, readv op sval); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ PUSH_I32(readv); \ break; \ @@ -483,39 +483,39 @@ LOAD_PTR(void *addr) CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(1); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = (uint64)(*(uint8 *)maddr); \ *(uint8 *)maddr = (uint8)(readv op sval); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##16_U) { \ CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(2); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = (uint64)LOAD_U16(maddr); \ STORE_U16(maddr, (uint16)(readv op sval)); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##32_U) { \ CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(4); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = (uint64)LOAD_U32(maddr); \ STORE_U32(maddr, (uint32)(readv op sval)); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ else { \ uint64 op_result; \ CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); \ CHECK_ATOMIC_MEMORY_ACCESS(8); \ \ - os_mutex_lock(&memory->mem_lock); \ + os_mutex_lock(&module->e->mem_lock); \ readv = (uint64)LOAD_I64(maddr); \ op_result = readv op sval; \ STORE_I64(maddr, op_result); \ - os_mutex_unlock(&memory->mem_lock); \ + os_mutex_unlock(&module->e->mem_lock); \ } \ PUSH_I64(readv); \ break; \ @@ -913,7 +913,7 @@ wasm_interp_call_func_native(WASMModuleInstance *module_inst, wasm_exec_env_set_cur_frame(exec_env, frame); - cur_func_index = (uint32)(cur_func - module_inst->functions); + cur_func_index = (uint32)(cur_func - module_inst->e->functions); bh_assert(cur_func_index < module_inst->module->import_function_count); native_func_pointer = module_inst->import_func_ptrs[cur_func_index]; @@ -1138,7 +1138,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, WASMFunctionInstance *cur_func, WASMInterpFrame *prev_frame) { - WASMMemoryInstance *memory = module->default_memory; + WASMMemoryInstance *memory = wasm_get_default_memory(module); #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ || WASM_ENABLE_BULK_MEMORY != 0 @@ -1147,7 +1147,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, memory ? num_bytes_per_page * memory->cur_page_count : 0; #endif uint8 *global_data = module->global_data; - WASMGlobalInstance *globals = module->globals, *global; + WASMGlobalInstance *globals = module->e ? module->e->globals : NULL; + WASMGlobalInstance *global; uint8 opcode_IMPDEP = WASM_OP_IMPDEP; WASMInterpFrame *frame = NULL; /* Points to this special opcode so as to jump to the @@ -1345,8 +1346,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; } - fidx = ((uint32 *)tbl_inst->base_addr)[val]; - if (fidx == (uint32)-1) { + fidx = tbl_inst->elems[val]; + if (fidx == NULL_REF) { wasm_set_exception(module, "uninitialized element"); goto got_exception; } @@ -1356,13 +1357,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, * another module. in that case, we don't validate * the elem value while loading */ - if (fidx >= module->function_count) { + if (fidx >= module->e->function_count) { wasm_set_exception(module, "unknown function"); goto got_exception; } /* always call module own functions */ - cur_func = module->functions + fidx; + cur_func = module->e->functions + fidx; if (cur_func->is_import_func) cur_func_type = cur_func->u.func_import->func_type; @@ -1437,7 +1438,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; } - PUSH_I32(((uint32 *)tbl_inst->base_addr)[elem_idx]); + PUSH_I32(tbl_inst->elems[elem_idx]); HANDLE_OP_END(); } @@ -1458,7 +1459,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; } - ((uint32 *)tbl_inst->base_addr)[elem_idx] = elem_val; + tbl_inst->elems[elem_idx] = elem_val; HANDLE_OP_END(); } @@ -1522,7 +1523,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_GET_GLOBAL) { global_idx = read_uint32(frame_ip); - bh_assert(global_idx < module->global_count); + bh_assert(global_idx < module->e->global_count); global = globals + global_idx; global_addr = get_global_addr(global_data, global); addr_ret = GET_OFFSET(); @@ -1533,7 +1534,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_GET_GLOBAL_64) { global_idx = read_uint32(frame_ip); - bh_assert(global_idx < module->global_count); + bh_assert(global_idx < module->e->global_count); global = globals + global_idx; global_addr = get_global_addr(global_data, global); addr_ret = GET_OFFSET(); @@ -1545,7 +1546,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_SET_GLOBAL) { global_idx = read_uint32(frame_ip); - bh_assert(global_idx < module->global_count); + bh_assert(global_idx < module->e->global_count); global = globals + global_idx; global_addr = get_global_addr(global_data, global); addr1 = GET_OFFSET(); @@ -1558,7 +1559,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, uint32 aux_stack_top; global_idx = read_uint32(frame_ip); - bh_assert(global_idx < module->global_count); + bh_assert(global_idx < module->e->global_count); global = globals + global_idx; global_addr = get_global_addr(global_data, global); aux_stack_top = frame_lp[GET_OFFSET()]; @@ -1576,8 +1577,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (module->module->aux_stack_top_global_index != (uint32)-1) { uint32 aux_stack_used = module->module->aux_stack_bottom - *(uint32 *)global_addr; - if (aux_stack_used > module->max_aux_stack_used) - module->max_aux_stack_used = aux_stack_used; + if (aux_stack_used > module->e->max_aux_stack_used) + module->e->max_aux_stack_used = aux_stack_used; } #endif HANDLE_OP_END(); @@ -1586,7 +1587,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_SET_GLOBAL_64) { global_idx = read_uint32(frame_ip); - bh_assert(global_idx < module->global_count); + bh_assert(global_idx < module->e->global_count); global = globals + global_idx; global_addr = get_global_addr(global_data, global); addr1 = GET_OFFSET(); @@ -1860,8 +1861,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, else { /* success, return previous page count */ frame_lp[addr_ret] = prev_page_count; - /* update memory instance ptr and memory size */ - memory = module->default_memory; + /* update memory size, no need to update memory ptr as + it isn't changed in wasm_enlarge_memory */ #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ || WASM_ENABLE_BULK_MEMORY != 0 @@ -3093,7 +3094,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, bh_memcpy_s( (uint8 *)tbl_inst - + offsetof(WASMTableInstance, base_addr) + + offsetof(WASMTableInstance, elems) + d * sizeof(uint32), (uint32)((tbl_inst->cur_size - d) * sizeof(uint32)), module->module->table_segments[elem_idx] @@ -3141,16 +3142,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, /* if s >= d, copy from front to back */ /* if s < d, copy from back to front */ /* merge all together */ - bh_memmove_s( - (uint8 *)dst_tbl_inst - + offsetof(WASMTableInstance, base_addr) - + d * sizeof(uint32), - (uint32)((dst_tbl_inst->cur_size - d) - * sizeof(uint32)), - (uint8 *)src_tbl_inst - + offsetof(WASMTableInstance, base_addr) - + s * sizeof(uint32), - (uint32)(n * sizeof(uint32))); + bh_memmove_s((uint8 *)dst_tbl_inst + + offsetof(WASMTableInstance, elems) + + d * sizeof(uint32), + (uint32)((dst_tbl_inst->cur_size - d) + * sizeof(uint32)), + (uint8 *)src_tbl_inst + + offsetof(WASMTableInstance, elems) + + s * sizeof(uint32), + (uint32)(n * sizeof(uint32))); break; } case WASM_OP_TABLE_GROW: @@ -3211,7 +3211,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, } for (; n != 0; i++, n--) { - ((uint32 *)(tbl_inst->base_addr))[i] = fill_val; + tbl_inst->elems[i] = fill_val; } break; @@ -3305,23 +3305,23 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (opcode == WASM_OP_ATOMIC_I32_LOAD8_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); CHECK_ATOMIC_MEMORY_ACCESS(1); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint32)(*(uint8 *)maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_I32_LOAD16_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); CHECK_ATOMIC_MEMORY_ACCESS(2); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint32)LOAD_U16(maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); CHECK_ATOMIC_MEMORY_ACCESS(4); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = LOAD_I32(maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } PUSH_I32(readv); @@ -3340,30 +3340,30 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (opcode == WASM_OP_ATOMIC_I64_LOAD8_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); CHECK_ATOMIC_MEMORY_ACCESS(1); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)(*(uint8 *)maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_I64_LOAD16_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); CHECK_ATOMIC_MEMORY_ACCESS(2); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)LOAD_U16(maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_I64_LOAD32_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); CHECK_ATOMIC_MEMORY_ACCESS(4); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)LOAD_U32(maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); CHECK_ATOMIC_MEMORY_ACCESS(8); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = LOAD_I64(maddr); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } PUSH_I64(readv); @@ -3381,23 +3381,23 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (opcode == WASM_OP_ATOMIC_I32_STORE8) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); CHECK_ATOMIC_MEMORY_ACCESS(1); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); *(uint8 *)maddr = (uint8)sval; - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_I32_STORE16) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); CHECK_ATOMIC_MEMORY_ACCESS(2); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); STORE_U16(maddr, (uint16)sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); CHECK_ATOMIC_MEMORY_ACCESS(4); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); STORE_U32(maddr, sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } break; } @@ -3415,30 +3415,30 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (opcode == WASM_OP_ATOMIC_I64_STORE8) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); CHECK_ATOMIC_MEMORY_ACCESS(1); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); *(uint8 *)maddr = (uint8)sval; - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_I64_STORE16) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); CHECK_ATOMIC_MEMORY_ACCESS(2); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); STORE_U16(maddr, (uint16)sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_I64_STORE32) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); CHECK_ATOMIC_MEMORY_ACCESS(4); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); STORE_U32(maddr, (uint32)sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); CHECK_ATOMIC_MEMORY_ACCESS(8); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); STORE_I64(maddr, sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } break; } @@ -3458,32 +3458,32 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, CHECK_ATOMIC_MEMORY_ACCESS(1); expect = (uint8)expect; - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint32)(*(uint8 *)maddr); if (readv == expect) *(uint8 *)maddr = (uint8)(sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); CHECK_ATOMIC_MEMORY_ACCESS(2); expect = (uint16)expect; - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint32)LOAD_U16(maddr); if (readv == expect) STORE_U16(maddr, (uint16)(sval)); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); CHECK_ATOMIC_MEMORY_ACCESS(4); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = LOAD_I32(maddr); if (readv == expect) STORE_U32(maddr, sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } PUSH_I32(readv); break; @@ -3504,44 +3504,44 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, CHECK_ATOMIC_MEMORY_ACCESS(1); expect = (uint8)expect; - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)(*(uint8 *)maddr); if (readv == expect) *(uint8 *)maddr = (uint8)(sval); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); CHECK_ATOMIC_MEMORY_ACCESS(2); expect = (uint16)expect; - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)LOAD_U16(maddr); if (readv == expect) STORE_U16(maddr, (uint16)(sval)); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U) { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); CHECK_ATOMIC_MEMORY_ACCESS(4); expect = (uint32)expect; - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)LOAD_U32(maddr); if (readv == expect) STORE_U32(maddr, (uint32)(sval)); - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } else { CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); CHECK_ATOMIC_MEMORY_ACCESS(8); - os_mutex_lock(&memory->mem_lock); + os_mutex_lock(&module->e->mem_lock); readv = (uint64)LOAD_I64(maddr); if (readv == expect) { STORE_I64(maddr, sval); } - os_mutex_unlock(&memory->mem_lock); + os_mutex_unlock(&module->e->mem_lock); } PUSH_I64(readv); break; @@ -3575,12 +3575,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #endif fidx = read_uint32(frame_ip); #if WASM_ENABLE_MULTI_MODULE != 0 - if (fidx >= module->function_count) { + if (fidx >= module->e->function_count) { wasm_set_exception(module, "unknown function"); goto got_exception; } #endif - cur_func = module->functions + fidx; + cur_func = module->e->functions + fidx; goto call_func_from_interp; } @@ -3592,12 +3592,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #endif fidx = read_uint32(frame_ip); #if WASM_ENABLE_MULTI_MODULE != 0 - if (fidx >= module->function_count) { + if (fidx >= module->e->function_count) { wasm_set_exception(module, "unknown function"); goto got_exception; } #endif - cur_func = module->functions + fidx; + cur_func = module->e->functions + fidx; goto call_func_from_return_call; } #endif /* WASM_ENABLE_TAIL_CALL */ @@ -3782,8 +3782,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, cur_func = frame->function; UPDATE_ALL_FROM_FRAME(); - /* update memory instance ptr and memory size */ - memory = module->default_memory; + /* update memory size, no need to update memory ptr as + it isn't changed in wasm_enlarge_memory */ #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ || WASM_ENABLE_BULK_MEMORY != 0 diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index e5941b7b..6ec7caab 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -18,6 +18,9 @@ #include "../fast-jit/jit_compiler.h" #include "../fast-jit/jit_codecache.h" #endif +#if WASM_ENABLE_JIT != 0 +#include "../compilation/aot_llvm.h" +#endif /* Read a value of given type from the address pointed to by the given pointer and increase the pointer to the position just after the @@ -2923,7 +2926,6 @@ fail: return false; } -#if WASM_ENABLE_FAST_JIT != 0 static void calculate_global_data_offset(WASMModule *module) { @@ -2933,17 +2935,127 @@ calculate_global_data_offset(WASMModule *module) for (i = 0; i < module->import_global_count; i++) { WASMGlobalImport *import_global = &((module->import_globals + i)->u.global); +#if WASM_ENABLE_FAST_JIT != 0 import_global->data_offset = data_offset; +#endif data_offset += wasm_value_type_size(import_global->type); } for (i = 0; i < module->global_count; i++) { WASMGlobal *global = module->globals + i; +#if WASM_ENABLE_FAST_JIT != 0 global->data_offset = data_offset; +#endif data_offset += wasm_value_type_size(global->type); } + + module->global_data_size = data_offset; } + +#if WASM_ENABLE_JIT != 0 +static bool +compile_llvm_jit_functions(WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + AOTCompOption option = { 0 }; + char func_name[32], *aot_last_error; + uint64 size; + uint32 i; + + size = sizeof(void *) * (uint64)module->function_count; + if (size > 0 + && !(module->func_ptrs = + loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + + module->comp_data = aot_create_comp_data(module); + if (!module->comp_data) { + aot_last_error = aot_get_last_error(); + bh_assert(aot_last_error != NULL); + set_error_buf(error_buf, error_buf_size, aot_last_error); + return false; + } + + option.is_jit_mode = true; + option.opt_level = 3; + option.size_level = 3; +#if WASM_ENABLE_BULK_MEMORY != 0 + option.enable_bulk_memory = true; #endif +#if WASM_ENABLE_THREAD_MGR != 0 + option.enable_thread_mgr = true; +#endif +#if WASM_ENABLE_TAIL_CALL != 0 + option.enable_tail_call = true; +#endif +#if WASM_ENABLE_SIMD != 0 + option.enable_simd = true; +#endif +#if WASM_ENABLE_REF_TYPES != 0 + option.enable_ref_types = true; +#endif + option.enable_aux_stack_check = true; +#if (WASM_ENABLE_PERF_PROFILING != 0) || (WASM_ENABLE_DUMP_CALL_STACK != 0) + option.enable_aux_stack_frame = true; +#endif + + module->comp_ctx = aot_create_comp_context(module->comp_data, &option); + if (!module->comp_ctx) { + aot_last_error = aot_get_last_error(); + bh_assert(aot_last_error != NULL); + set_error_buf(error_buf, error_buf_size, aot_last_error); + return false; + } + + if (!aot_compile_wasm(module->comp_ctx)) { + aot_last_error = aot_get_last_error(); + bh_assert(aot_last_error != NULL); + set_error_buf(error_buf, error_buf_size, aot_last_error); + return false; + } + +#if WASM_ENABLE_LAZY_JIT != 0 + for (i = 0; i < module->comp_data->func_count; i++) { + LLVMErrorRef error; + LLVMOrcJITTargetAddress func_addr = 0; + + snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i); + if ((error = LLVMOrcLLJITLookup(module->comp_ctx->orc_lazyjit, + &func_addr, func_name))) { + char *err_msg = LLVMGetErrorMessage(error); + set_error_buf_v(error_buf, error_buf_size, + "failed to compile orc jit function: %s", err_msg); + LLVMDisposeErrorMessage(err_msg); + return false; + } + /** + * No need to lock the func_ptr[func_idx] here as it is basic + * data type, the load/store for it can be finished by one cpu + * instruction, and there can be only one cpu instruction + * loading/storing at the same time. + */ + module->func_ptrs[i] = (void *)func_addr; + module->functions[i]->llvm_jit_func_ptr = (void *)func_addr; + } +#else + /* Resolve function addresses */ + bh_assert(module->comp_ctx->exec_engine); + for (i = 0; i < module->comp_data->func_count; i++) { + snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i); + if (!(module->func_ptrs[i] = (void *)LLVMGetFunctionAddress( + module->comp_ctx->exec_engine, func_name))) { + set_error_buf(error_buf, error_buf_size, + "failed to compile llvm mc jit function"); + return false; + } + module->functions[i]->llvm_jit_func_ptr = module->func_ptrs[i]; + } +#endif /* end of WASM_ENABLE_LAZY_JIT != 0 */ + + return true; +} +#endif /* end of WASM_ENABLE_JIT != 0 */ static bool wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func, @@ -3352,9 +3464,9 @@ load_from_sections(WASMModule *module, WASMSection *sections, #endif } -#if WASM_ENABLE_FAST_JIT != 0 calculate_global_data_offset(module); +#if WASM_ENABLE_FAST_JIT != 0 if (module->function_count && !(module->fast_jit_func_ptrs = loader_malloc(sizeof(void *) * module->function_count, @@ -3367,6 +3479,12 @@ load_from_sections(WASMModule *module, WASMSection *sections, } #endif +#if WASM_ENABLE_JIT != 0 + if (!compile_llvm_jit_functions(module, error_buf, error_buf_size)) { + return false; + } +#endif /* end of WASM_ENABLE_JIT != 0 */ + #if WASM_ENABLE_MEMORY_TRACING != 0 wasm_runtime_dump_module_mem_consumption((WASMModuleCommon *)module); #endif @@ -3900,6 +4018,15 @@ wasm_loader_unload(WASMModule *module) } #endif +#if WASM_ENABLE_JIT != 0 + if (module->func_ptrs) + wasm_runtime_free(module->func_ptrs); + if (module->comp_ctx) + aot_destroy_comp_context(module->comp_ctx); + if (module->comp_data) + aot_destroy_comp_data(module->comp_data); +#endif + wasm_runtime_free(module); } @@ -6756,8 +6883,7 @@ re_scan: } block_type.is_value_type = false; block_type.u.type = module->types[type_index]; -#if WASM_ENABLE_FAST_INTERP == 0 && WASM_ENABLE_WAMR_COMPILER == 0 \ - && WASM_ENABLE_JIT == 0 +#if WASM_ENABLE_FAST_INTERP == 0 /* If block use type index as block type, change the opcode * to new extended opcode so that interpreter can resolve * the block quickly. diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index a2f58956..1773e623 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -15,6 +15,9 @@ #include "../fast-jit/jit_compiler.h" #include "../fast-jit/jit_codecache.h" #endif +#if WASM_ENABLE_JIT != 0 +#include "../compilation/aot_llvm.h" +#endif /* Read a value of given type from the address pointed to by the given pointer and increase the pointer to the position just after the @@ -961,6 +964,11 @@ load_function_section(const uint8 *buf, const uint8 *buf_end, read_leb_uint32(p, p_end, type_index); bh_assert(type_index < module->type_count); +#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) + type_index = wasm_get_smallest_type_idx( + module->types, module->type_count, type_index); +#endif + read_leb_uint32(p_code, buf_code_end, code_size); bh_assert(code_size > 0 && p_code + code_size <= buf_code_end); @@ -1749,6 +1757,137 @@ load_user_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, return true; } +static void +calculate_global_data_offset(WASMModule *module) +{ + uint32 i, data_offset; + + data_offset = 0; + for (i = 0; i < module->import_global_count; i++) { + WASMGlobalImport *import_global = + &((module->import_globals + i)->u.global); +#if WASM_ENABLE_FAST_JIT != 0 + import_global->data_offset = data_offset; +#endif + data_offset += wasm_value_type_size(import_global->type); + } + + for (i = 0; i < module->global_count; i++) { + WASMGlobal *global = module->globals + i; +#if WASM_ENABLE_FAST_JIT != 0 + global->data_offset = data_offset; +#endif + data_offset += wasm_value_type_size(global->type); + } + + module->global_data_size = data_offset; +} + +#if WASM_ENABLE_JIT != 0 +static bool +compile_llvm_jit_functions(WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + AOTCompOption option = { 0 }; + char func_name[32], *aot_last_error; + uint64 size; + uint32 i; + + size = sizeof(void *) * (uint64)module->function_count; + if (size > 0 + && !(module->func_ptrs = + loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + + module->comp_data = aot_create_comp_data(module); + if (!module->comp_data) { + aot_last_error = aot_get_last_error(); + bh_assert(aot_last_error != NULL); + set_error_buf(error_buf, error_buf_size, aot_last_error); + return false; + } + + option.is_jit_mode = true; + option.opt_level = 3; + option.size_level = 3; +#if WASM_ENABLE_BULK_MEMORY != 0 + option.enable_bulk_memory = true; +#endif +#if WASM_ENABLE_THREAD_MGR != 0 + option.enable_thread_mgr = true; +#endif +#if WASM_ENABLE_TAIL_CALL != 0 + option.enable_tail_call = true; +#endif +#if WASM_ENABLE_SIMD != 0 + option.enable_simd = true; +#endif +#if WASM_ENABLE_REF_TYPES != 0 + option.enable_ref_types = true; +#endif + option.enable_aux_stack_check = true; +#if (WASM_ENABLE_PERF_PROFILING != 0) || (WASM_ENABLE_DUMP_CALL_STACK != 0) + option.enable_aux_stack_frame = true; +#endif + + module->comp_ctx = aot_create_comp_context(module->comp_data, &option); + if (!module->comp_ctx) { + aot_last_error = aot_get_last_error(); + bh_assert(aot_last_error != NULL); + set_error_buf(error_buf, error_buf_size, aot_last_error); + return false; + } + + if (!aot_compile_wasm(module->comp_ctx)) { + aot_last_error = aot_get_last_error(); + bh_assert(aot_last_error != NULL); + set_error_buf(error_buf, error_buf_size, aot_last_error); + return false; + } + +#if WASM_ENABLE_LAZY_JIT != 0 + for (i = 0; i < module->comp_data->func_count; i++) { + LLVMErrorRef error; + LLVMOrcJITTargetAddress func_addr = 0; + + snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i); + if ((error = LLVMOrcLLJITLookup(module->comp_ctx->orc_lazyjit, + &func_addr, func_name))) { + char *err_msg = LLVMGetErrorMessage(error); + set_error_buf_v(error_buf, error_buf_size, + "failed to compile orc jit function: %s", err_msg); + LLVMDisposeErrorMessage(err_msg); + return false; + } + /** + * No need to lock the func_ptr[func_idx] here as it is basic + * data type, the load/store for it can be finished by one cpu + * instruction, and there can be only one cpu instruction + * loading/storing at the same time. + */ + module->func_ptrs[i] = (void *)func_addr; + module->functions[i]->llvm_jit_func_ptr = (void *)func_addr; + } +#else + /* Resolve function addresses */ + bh_assert(module->comp_ctx->exec_engine); + for (i = 0; i < module->comp_data->func_count; i++) { + snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i); + if (!(module->func_ptrs[i] = (void *)LLVMGetFunctionAddress( + module->comp_ctx->exec_engine, func_name))) { + set_error_buf(error_buf, error_buf_size, + "failed to compile llvm mc jit function"); + return false; + } + module->functions[i]->llvm_jit_func_ptr = module->func_ptrs[i]; + } +#endif /* end of WASM_ENABLE_LAZY_JIT != 0 */ + + return true; +} +#endif /* end of WASM_ENABLE_JIT != 0 */ + #if WASM_ENABLE_REF_TYPES != 0 static bool get_table_elem_type(const WASMModule *module, uint32 table_idx, @@ -2185,6 +2324,8 @@ load_from_sections(WASMModule *module, WASMSection *sections, } } + calculate_global_data_offset(module); + #if WASM_ENABLE_FAST_JIT != 0 if (!(module->fast_jit_func_ptrs = loader_malloc(sizeof(void *) * module->function_count, error_buf, @@ -2197,6 +2338,12 @@ load_from_sections(WASMModule *module, WASMSection *sections, } #endif +#if WASM_ENABLE_JIT != 0 + if (!compile_llvm_jit_functions(module, error_buf, error_buf_size)) { + return false; + } +#endif /* end of WASM_ENABLE_JIT != 0 */ + #if WASM_ENABLE_MEMORY_TRACING != 0 wasm_runtime_dump_module_mem_consumption(module); #endif @@ -2526,6 +2673,15 @@ wasm_loader_unload(WASMModule *module) } #endif +#if WASM_ENABLE_JIT != 0 + if (module->func_ptrs) + wasm_runtime_free(module->func_ptrs); + if (module->comp_ctx) + aot_destroy_comp_context(module->comp_ctx); + if (module->comp_data) + aot_destroy_comp_data(module->comp_data); +#endif + wasm_runtime_free(module); } @@ -5035,8 +5191,7 @@ re_scan: bh_assert(type_index < module->type_count); block_type.is_value_type = false; block_type.u.type = module->types[type_index]; -#if WASM_ENABLE_FAST_INTERP == 0 && WASM_ENABLE_WAMR_COMPILER == 0 \ - && WASM_ENABLE_JIT == 0 +#if WASM_ENABLE_FAST_INTERP == 0 /* If block use type index as block type, change the opcode * to new extended opcode so that interpreter can resolve * the block quickly. diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index f085e785..7d00b5ea 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -19,6 +19,9 @@ #if WASM_ENABLE_DEBUG_INTERP != 0 #include "../libraries/debug-engine/debug_engine.h" #endif +#if WASM_ENABLE_JIT != 0 +#include "../aot/aot_runtime.h" +#endif static void set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) @@ -87,7 +90,7 @@ static WASMModuleInstance * get_sub_module_inst(const WASMModuleInstance *parent_module_inst, const WASMModule *sub_module) { - bh_list *sub_module_inst_list = parent_module_inst->sub_module_inst_list; + bh_list *sub_module_inst_list = parent_module_inst->e->sub_module_inst_list; WASMSubModInstNode *node = bh_list_first_elem(sub_module_inst_list); while (node && sub_module != node->module_inst->module) { @@ -126,7 +129,6 @@ memories_deinstantiate(WASMModuleInstance *module_inst, if (ref_count > 0) continue; } - os_mutex_destroy(&memories[i]->mem_lock); #endif if (memories[i]->heap_handle) { mem_allocator_destroy(memories[i]->heap_handle); @@ -146,7 +148,6 @@ memories_deinstantiate(WASMModuleInstance *module_inst, 8 * (uint64)BH_GB); #endif } - wasm_runtime_free(memories[i]); } } wasm_runtime_free(memories); @@ -155,13 +156,12 @@ memories_deinstantiate(WASMModuleInstance *module_inst, } static WASMMemoryInstance * -memory_instantiate(WASMModuleInstance *module_inst, uint32 num_bytes_per_page, - uint32 init_page_count, uint32 max_page_count, - uint32 heap_size, uint32 flags, char *error_buf, - uint32 error_buf_size) +memory_instantiate(WASMModuleInstance *module_inst, WASMMemoryInstance *memory, + uint32 num_bytes_per_page, uint32 init_page_count, + uint32 max_page_count, uint32 heap_size, uint32 flags, + char *error_buf, uint32 error_buf_size) { WASMModule *module = module_inst->module; - WASMMemoryInstance *memory; uint64 memory_data_size; uint32 heap_offset = num_bytes_per_page * init_page_count; uint32 inc_page_count, aux_heap_base, global_idx; @@ -250,10 +250,10 @@ memory_instantiate(WASMModuleInstance *module_inst, uint32 num_bytes_per_page, /* Adjust __heap_base global value */ global_idx = module->aux_heap_base_global_index; - bh_assert(module_inst->globals - && global_idx < module_inst->global_count); + bh_assert(module_inst->e->globals + && global_idx < module_inst->e->global_count); global_addr = module_inst->global_data - + module_inst->globals[global_idx].data_offset; + + module_inst->e->globals[global_idx].data_offset; *(uint32 *)global_addr = aux_heap_base; LOG_VERBOSE("Reset __heap_base global to %u", aux_heap_base); } @@ -296,12 +296,6 @@ memory_instantiate(WASMModuleInstance *module_inst, uint32 num_bytes_per_page, #endif bh_assert(memory_data_size <= 4 * (uint64)BH_GB); - /* Allocate memory space, addr data and global data */ - if (!(memory = runtime_malloc((uint64)sizeof(WASMMemoryInstance), error_buf, - error_buf_size))) { - return NULL; - } - #ifndef OS_ENABLE_HW_BOUND_CHECK if (memory_data_size > 0 && !(memory->memory_data = @@ -370,44 +364,40 @@ memory_instantiate(WASMModuleInstance *module_inst, uint32 num_bytes_per_page, } } -#if WASM_ENABLE_FAST_JIT != 0 +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 if (memory_data_size > 0) { #if UINTPTR_MAX == UINT64_MAX - memory->mem_bound_check_1byte = memory_data_size - 1; - memory->mem_bound_check_2bytes = memory_data_size - 2; - memory->mem_bound_check_4bytes = memory_data_size - 4; - memory->mem_bound_check_8bytes = memory_data_size - 8; - memory->mem_bound_check_16bytes = memory_data_size - 16; + memory->mem_bound_check_1byte.u64 = memory_data_size - 1; + memory->mem_bound_check_2bytes.u64 = memory_data_size - 2; + memory->mem_bound_check_4bytes.u64 = memory_data_size - 4; + memory->mem_bound_check_8bytes.u64 = memory_data_size - 8; + memory->mem_bound_check_16bytes.u64 = memory_data_size - 16; #else - memory->mem_bound_check_1byte = (uint32)memory_data_size - 1; - memory->mem_bound_check_2bytes = (uint32)memory_data_size - 2; - memory->mem_bound_check_4bytes = (uint32)memory_data_size - 4; - memory->mem_bound_check_8bytes = (uint32)memory_data_size - 8; - memory->mem_bound_check_16bytes = (uint32)memory_data_size - 16; + memory->mem_bound_check_1byte.u32[0] = (uint32)memory_data_size - 1; + memory->mem_bound_check_2bytes.u32[0] = (uint32)memory_data_size - 2; + memory->mem_bound_check_4bytes.u32[0] = (uint32)memory_data_size - 4; + memory->mem_bound_check_8bytes.u32[0] = (uint32)memory_data_size - 8; + memory->mem_bound_check_16bytes.u32[0] = (uint32)memory_data_size - 16; #endif } #endif #if WASM_ENABLE_SHARED_MEMORY != 0 - if (0 != os_mutex_init(&memory->mem_lock)) { - set_error_buf(error_buf, error_buf_size, "init mutex failed"); - goto fail4; - } if (is_shared_memory) { memory->is_shared = true; if (!shared_memory_set_memory_inst( (WASMModuleCommon *)module_inst->module, (WASMMemoryInstanceCommon *)memory)) { set_error_buf(error_buf, error_buf_size, "allocate memory failed"); - goto fail5; + goto fail4; } } #endif + LOG_VERBOSE("Memory instantiate success."); return memory; + #if WASM_ENABLE_SHARED_MEMORY != 0 -fail5: - os_mutex_destroy(&memory->mem_lock); fail4: if (heap_size > 0) mem_allocator_destroy(memory->heap_handle); @@ -426,8 +416,7 @@ fail2: os_munmap(mapped_mem, map_size); #endif fail1: - wasm_runtime_free(memory); - return NULL; + return memory; } /** @@ -449,9 +438,11 @@ memories_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, return NULL; } + memory = module_inst->global_table_data.memory_instances; + /* instantiate memories from import section */ import = module->import_memories; - for (i = 0; i < module->import_memory_count; i++, import++) { + for (i = 0; i < module->import_memory_count; i++, import++, memory++) { uint32 num_bytes_per_page = import->u.memory.num_bytes_per_page; uint32 init_page_count = import->u.memory.init_page_count; uint32 max_page_count = import->u.memory.max_page_count; @@ -469,7 +460,7 @@ memories_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, return NULL; } - if (!(memory = memories[mem_index++] = wasm_lookup_memory( + if (!(memories[mem_index++] = wasm_lookup_memory( module_inst_linked, import->u.memory.field_name))) { set_error_buf(error_buf, error_buf_size, "unknown memory"); memories_deinstantiate(module_inst, memories, memory_count); @@ -479,8 +470,8 @@ memories_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, else #endif { - if (!(memory = memories[mem_index++] = memory_instantiate( - module_inst, num_bytes_per_page, init_page_count, + if (!(memories[mem_index++] = memory_instantiate( + module_inst, memory, num_bytes_per_page, init_page_count, max_page_count, actual_heap_size, flags, error_buf, error_buf_size))) { memories_deinstantiate(module_inst, memories, memory_count); @@ -490,9 +481,9 @@ memories_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, } /* instantiate memories from memory section */ - for (i = 0; i < module->memory_count; i++) { - if (!(memory = memories[mem_index++] = memory_instantiate( - module_inst, module->memories[i].num_bytes_per_page, + for (i = 0; i < module->memory_count; i++, memory++) { + if (!(memories[mem_index++] = memory_instantiate( + module_inst, memory, module->memories[i].num_bytes_per_page, module->memories[i].init_page_count, module->memories[i].max_page_count, heap_size, module->memories[i].flags, error_buf, error_buf_size))) { @@ -510,15 +501,16 @@ memories_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, * Destroy table instances. */ static void -tables_deinstantiate(WASMTableInstance **tables, uint32 count) +tables_deinstantiate(WASMModuleInstance *module_inst) { - uint32 i; - if (tables) { - for (i = 0; i < count; i++) - if (tables[i]) - wasm_runtime_free(tables[i]); - wasm_runtime_free(tables); + if (module_inst->tables) { + wasm_runtime_free(module_inst->tables); } +#if WASM_ENABLE_MULTI_MODULE != 0 + if (module_inst->e->table_insts_linked) { + wasm_runtime_free(module_inst->e->table_insts_linked); + } +#endif } /** @@ -526,18 +518,32 @@ tables_deinstantiate(WASMTableInstance **tables, uint32 count) */ static WASMTableInstance ** tables_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, - char *error_buf, uint32 error_buf_size) + WASMTableInstance *first_table, char *error_buf, + uint32 error_buf_size) { WASMImport *import; - uint32 table_index = 0, i, - table_count = module->import_table_count + module->table_count; - uint64 total_size = sizeof(WASMTableInstance *) * (uint64)table_count; - WASMTableInstance **tables, *table; + uint32 table_index = 0, i; + uint32 table_count = module->import_table_count + module->table_count; + uint64 total_size = (uint64)sizeof(WASMTableInstance *) * table_count; + WASMTableInstance **tables, *table = first_table; +#if WASM_ENABLE_MULTI_MODULE != 0 + uint64 total_size_of_tables_linked = + (uint64)sizeof(WASMTableInstance *) * module->import_table_count; + WASMTableInstance **table_linked = NULL; +#endif if (!(tables = runtime_malloc(total_size, error_buf, error_buf_size))) { return NULL; } +#if WASM_ENABLE_MULTI_MODULE != 0 + if (module->import_table_count > 0 + && !(module_inst->e->table_insts_linked = table_linked = runtime_malloc( + total_size_of_tables_linked, error_buf, error_buf_size))) { + goto fail; + } +#endif + /* instantiate tables from import section */ import = module->import_tables; for (i = 0; i < module->import_table_count; i++, import++) { @@ -550,18 +556,16 @@ tables_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, if (!(module_inst_linked = get_sub_module_inst( module_inst, import->u.table.import_module))) { set_error_buf(error_buf, error_buf_size, "unknown table"); - tables_deinstantiate(tables, table_count); - return NULL; + goto fail; } if (!(table_inst_linked = wasm_lookup_table( module_inst_linked, import->u.table.field_name))) { set_error_buf(error_buf, error_buf_size, "unknown table"); - tables_deinstantiate(tables, table_count); - return NULL; + goto fail; } - total_size = offsetof(WASMTableInstance, base_addr); + total_size = offsetof(WASMTableInstance, elems); } else #endif @@ -572,40 +576,39 @@ tables_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, : import->u.table.init_size; /* it is a built-in table, every module has its own */ - total_size = offsetof(WASMTableInstance, base_addr); + total_size = offsetof(WASMTableInstance, elems); total_size += (uint64)max_size_fixed * sizeof(uint32); } - if (!(table = tables[table_index++] = - runtime_malloc(total_size, error_buf, error_buf_size))) { - tables_deinstantiate(tables, table_count); - return NULL; - } + tables[table_index++] = table; /* Set all elements to -1 to mark them as uninitialized elements */ memset(table, -1, (uint32)total_size); #if WASM_ENABLE_MULTI_MODULE != 0 - table->table_inst_linked = table_inst_linked; + *table_linked = table_inst_linked; if (table_inst_linked != NULL) { - table->elem_type = table_inst_linked->elem_type; table->cur_size = table_inst_linked->cur_size; table->max_size = table_inst_linked->max_size; } else #endif { - table->elem_type = import->u.table.elem_type; table->cur_size = import->u.table.init_size; table->max_size = max_size_fixed; } + + table = (WASMTableInstance *)((uint8 *)table + (uint32)total_size); +#if WASM_ENABLE_MULTI_MODULE != 0 + table_linked++; +#endif } /* instantiate tables from table section */ for (i = 0; i < module->table_count; i++) { uint32 max_size_fixed = 0; - total_size = offsetof(WASMTableInstance, base_addr); + total_size = offsetof(WASMTableInstance, elems); #if WASM_ENABLE_MULTI_MODULE != 0 /* in case, a module which imports this table will grow it */ max_size_fixed = module->tables[i].max_size; @@ -616,25 +619,24 @@ tables_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, #endif total_size += sizeof(uint32) * (uint64)max_size_fixed; - if (!(table = tables[table_index++] = - runtime_malloc(total_size, error_buf, error_buf_size))) { - tables_deinstantiate(tables, table_count); - return NULL; - } + tables[table_index++] = table; /* Set all elements to -1 to mark them as uninitialized elements */ memset(table, -1, (uint32)total_size); - table->elem_type = module->tables[i].elem_type; table->cur_size = module->tables[i].init_size; table->max_size = max_size_fixed; -#if WASM_ENABLE_MULTI_MODULE != 0 - table->table_inst_linked = NULL; -#endif + + table = (WASMTableInstance *)((uint8 *)table + (uint32)total_size); } bh_assert(table_index == table_count); (void)module_inst; return tables; +#if WASM_ENABLE_MULTI_MODULE != 0 +fail: + wasm_runtime_free(tables); + return NULL; +#endif } /** @@ -783,8 +785,7 @@ check_global_init_expr(const WASMModule *module, uint32 global_index, */ static WASMGlobalInstance * globals_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, - uint32 *p_global_data_size, char *error_buf, - uint32 error_buf_size) + char *error_buf, uint32 error_buf_size) { WASMImport *import; uint32 global_data_offset = 0; @@ -831,6 +832,9 @@ globals_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, &(global_import->global_data_linked), sizeof(WASMValue)); } +#if WASM_ENABLE_FAST_JIT != 0 + bh_assert(global_data_offset == global_import->data_offset); +#endif global->data_offset = global_data_offset; global_data_offset += wasm_value_type_size(global->type); @@ -843,8 +847,10 @@ globals_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, global->type = module->globals[i].type; global->is_mutable = module->globals[i].is_mutable; +#if WASM_ENABLE_FAST_JIT != 0 + bh_assert(global_data_offset == module->globals[i].data_offset); +#endif global->data_offset = global_data_offset; - global_data_offset += wasm_value_type_size(global->type); if (init_expr->init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) { @@ -871,7 +877,7 @@ globals_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, } bh_assert((uint32)(global - globals) == global_count); - *p_global_data_size = global_data_offset; + bh_assert(global_data_offset == module->global_data_size); (void)module_inst; return globals; fail: @@ -928,7 +934,7 @@ export_functions_instantiate(const WASMModule *module, for (i = 0; i < module->export_count; i++, export ++) if (export->kind == EXPORT_KIND_FUNC) { export_func->name = export->name; - export_func->function = &module_inst->functions[export->index]; + export_func->function = &module_inst->e->functions[export->index]; export_func++; } @@ -964,7 +970,7 @@ export_globals_instantiate(const WASMModule *module, for (i = 0; i < module->export_count; i++, export ++) if (export->kind == EXPORT_KIND_GLOBAL) { export_global->name = export->name; - export_global->global = &module_inst->globals[export->index]; + export_global->global = &module_inst->e->globals[export->index]; export_global++; } @@ -1034,7 +1040,7 @@ execute_memory_init_function(WASMModuleInstance *module_inst) static bool execute_start_function(WASMModuleInstance *module_inst) { - WASMFunctionInstance *func = module_inst->start_function; + WASMFunctionInstance *func = module_inst->e->start_function; if (!func) return true; @@ -1129,7 +1135,7 @@ sub_module_instantiate(WASMModule *module, WASMModuleInstance *module_inst, uint32 stack_size, uint32 heap_size, char *error_buf, uint32 error_buf_size) { - bh_list *sub_module_inst_list = module_inst->sub_module_inst_list; + bh_list *sub_module_inst_list = module_inst->e->sub_module_inst_list; WASMRegisteredModule *sub_module_list_node = bh_list_first_elem(module->import_module_list); @@ -1205,7 +1211,7 @@ sub_module_instantiate(WASMModule *module, WASMModuleInstance *module_inst, static void sub_module_deinstantiate(WASMModuleInstance *module_inst) { - bh_list *list = module_inst->sub_module_inst_list; + bh_list *list = module_inst->e->sub_module_inst_list; WASMSubModInstNode *node = bh_list_first_elem(list); while (node) { WASMSubModInstNode *next_node = bh_list_elem_next(node); @@ -1262,7 +1268,37 @@ check_linked_symbol(WASMModuleInstance *module_inst, char *error_buf, return true; } -#if WASM_ENABLE_FAST_JIT != 0 +#if WASM_ENABLE_JIT != 0 +static bool +init_func_ptrs(WASMModuleInstance *module_inst, WASMModule *module, + char *error_buf, uint32 error_buf_size) +{ + uint32 i; + void **func_ptrs; + uint64 total_size = (uint64)sizeof(void *) * module_inst->e->function_count; + + /* Allocate memory */ + if (!(func_ptrs = module_inst->func_ptrs = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + /* Set import function pointers */ + for (i = 0; i < module->import_function_count; i++, func_ptrs++) { + WASMFunctionImport *import_func = + &module->import_functions[i].u.function; + /* TODO: handle multi module */ + *func_ptrs = import_func->func_ptr_linked; + } + + /* Set defined function pointers */ + bh_memcpy_s(func_ptrs, sizeof(void *) * module->function_count, + module->func_ptrs, sizeof(void *) * module->function_count); + return true; +} +#endif /* end of WASM_ENABLE_JIT != 0 */ + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 static uint32 get_smallest_type_idx(WASMModule *module, WASMType *func_type) { @@ -1282,7 +1318,7 @@ init_func_type_indexes(WASMModuleInstance *module_inst, char *error_buf, uint32 error_buf_size) { uint32 i; - uint64 total_size = (uint64)sizeof(uint32) * module_inst->function_count; + uint64 total_size = (uint64)sizeof(uint32) * module_inst->e->function_count; /* Allocate memory */ if (!(module_inst->func_type_indexes = @@ -1290,8 +1326,8 @@ init_func_type_indexes(WASMModuleInstance *module_inst, char *error_buf, return false; } - for (i = 0; i < module_inst->function_count; i++) { - WASMFunctionInstance *func_inst = module_inst->functions + i; + for (i = 0; i < module_inst->e->function_count; i++) { + WASMFunctionInstance *func_inst = module_inst->e->functions + i; WASMType *func_type = func_inst->is_import_func ? func_inst->u.func_import->func_type : func_inst->u.func->func_type; @@ -1301,7 +1337,7 @@ init_func_type_indexes(WASMModuleInstance *module_inst, char *error_buf, return true; } -#endif +#endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 */ /** * Instantiate module @@ -1312,8 +1348,13 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, { WASMModuleInstance *module_inst; WASMGlobalInstance *globals = NULL, *global; - uint32 global_count, global_data_size = 0, i; - uint32 base_offset, length; + WASMTableInstance *first_table; + uint32 global_count, i; + uint32 base_offset, length, extra_info_offset; + uint32 module_inst_struct_size = + offsetof(WASMModuleInstance, global_table_data.bytes); + uint64 module_inst_mem_inst_size; + uint64 total_size, table_size = 0; uint8 *global_data, *global_data_end; #if WASM_ENABLE_MULTI_MODULE != 0 bool ret = false; @@ -1322,6 +1363,56 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, if (!module) return NULL; + /* Check the heap size */ + heap_size = align_uint(heap_size, 8); + if (heap_size > APP_HEAP_SIZE_MAX) + heap_size = APP_HEAP_SIZE_MAX; + + module_inst_mem_inst_size = + (uint64)sizeof(WASMMemoryInstance) + * (module->import_memory_count + module->memory_count); + +#if WASM_ENABLE_JIT != 0 + /* If the module dosen't have memory, reserve one mem_info space + with empty content to align with llvm jit compiler */ + if (module_inst_mem_inst_size == 0) + module_inst_mem_inst_size = (uint64)sizeof(WASMMemoryInstance); +#endif + + /* Size of module inst, memory instances and global data */ + total_size = (uint64)module_inst_struct_size + module_inst_mem_inst_size + + module->global_data_size; + + /* Calculate the size of table data */ + for (i = 0; i < module->import_table_count; i++) { + WASMTableImport *import_table = &module->import_tables[i].u.table; + table_size += offsetof(WASMTableInstance, elems); +#if WASM_ENABLE_MULTI_MODULE != 0 + table_size += (uint64)sizeof(uint32) * import_table->max_size; +#else + table_size += (uint64)sizeof(uint32) + * (import_table->possible_grow ? import_table->max_size + : import_table->init_size); +#endif + } + for (i = 0; i < module->table_count; i++) { + WASMTable *table = module->tables + i; + table_size += offsetof(WASMTableInstance, elems); +#if WASM_ENABLE_MULTI_MODULE != 0 + table_size += (uint64)sizeof(uint32) * table->max_size; +#else + table_size += + (uint64)sizeof(uint32) + * (table->possible_grow ? table->max_size : table->init_size); +#endif + } + total_size += table_size; + + /* The offset of WASMModuleInstanceExtra, make it 8-byte aligned */ + total_size = (total_size + 7LL) & ~7LL; + extra_info_offset = (uint32)total_size; + total_size += sizeof(WASMModuleInstanceExtra); + #if WASM_ENABLE_DEBUG_INTERP != 0 if (!is_sub_inst) { os_mutex_lock(&module->ref_count_lock); @@ -1335,14 +1426,10 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, } #endif - /* Check heap size */ - heap_size = align_uint(heap_size, 8); - if (heap_size > APP_HEAP_SIZE_MAX) - heap_size = APP_HEAP_SIZE_MAX; - - /* Allocate the memory */ - if (!(module_inst = runtime_malloc(sizeof(WASMModuleInstance), error_buf, - error_buf_size))) { + /* Allocate the memory for module instance with memory instances, + global data, table data appended at the end */ + if (!(module_inst = + runtime_malloc(total_size, error_buf, error_buf_size))) { #if WASM_ENABLE_DEBUG_INTERP != 0 if (!is_sub_inst) { os_mutex_lock(&module->ref_count_lock); @@ -1353,10 +1440,23 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, return NULL; } + module_inst->module_type = Wasm_Module_Bytecode; module_inst->module = module; + module_inst->e = + (WASMModuleInstanceExtra *)((uint8 *)module_inst + extra_info_offset); + +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (os_mutex_init(&module_inst->e->mem_lock) != 0) { + set_error_buf(error_buf, error_buf_size, + "create shared memory lock failed"); + goto fail; + } + module_inst->e->mem_lock_inited = true; +#endif #if WASM_ENABLE_MULTI_MODULE != 0 - module_inst->sub_module_inst_list = &module_inst->sub_module_inst_list_head; + module_inst->e->sub_module_inst_list = + &module_inst->e->sub_module_inst_list_head; ret = sub_module_instantiate(module, module_inst, stack_size, heap_size, error_buf, error_buf_size); if (!ret) { @@ -1375,59 +1475,62 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, /* Instantiate global firstly to get the mutable data size */ global_count = module->import_global_count + module->global_count; if (global_count - && !(globals = - globals_instantiate(module, module_inst, &global_data_size, - error_buf, error_buf_size))) { + && !(globals = globals_instantiate(module, module_inst, error_buf, + error_buf_size))) { goto fail; } - module_inst->global_count = global_count; - module_inst->globals = globals; + module_inst->e->global_count = global_count; + module_inst->e->globals = globals; + module_inst->global_data = (uint8 *)module_inst + module_inst_struct_size + + module_inst_mem_inst_size; + module_inst->global_data_size = module->global_data_size; + first_table = (WASMTableInstance *)(module_inst->global_data + + module->global_data_size); module_inst->memory_count = module->import_memory_count + module->memory_count; module_inst->table_count = module->import_table_count + module->table_count; - module_inst->function_count = + module_inst->e->function_count = module->import_function_count + module->function_count; /* export */ module_inst->export_func_count = get_export_count(module, EXPORT_KIND_FUNC); #if WASM_ENABLE_MULTI_MODULE != 0 - module_inst->export_tab_count = get_export_count(module, EXPORT_KIND_TABLE); - module_inst->export_mem_count = + module_inst->export_table_count = + get_export_count(module, EXPORT_KIND_TABLE); + module_inst->export_memory_count = get_export_count(module, EXPORT_KIND_MEMORY); - module_inst->export_glob_count = + module_inst->export_global_count = get_export_count(module, EXPORT_KIND_GLOBAL); #endif - if (global_count > 0) { - if (!(module_inst->global_data = runtime_malloc( - global_data_size, error_buf, error_buf_size))) { - goto fail; - } - } - /* Instantiate memories/tables/functions */ if ((module_inst->memory_count > 0 && !(module_inst->memories = memories_instantiate( module, module_inst, heap_size, error_buf, error_buf_size))) || (module_inst->table_count > 0 - && !(module_inst->tables = tables_instantiate( - module, module_inst, error_buf, error_buf_size))) - || (module_inst->function_count > 0 - && !(module_inst->functions = functions_instantiate( + && !(module_inst->tables = + tables_instantiate(module, module_inst, first_table, + error_buf, error_buf_size))) + || (module_inst->e->function_count > 0 + && !(module_inst->e->functions = functions_instantiate( module, module_inst, error_buf, error_buf_size))) || (module_inst->export_func_count > 0 && !(module_inst->export_functions = export_functions_instantiate( module, module_inst, module_inst->export_func_count, error_buf, error_buf_size))) #if WASM_ENABLE_MULTI_MODULE != 0 - || (module_inst->export_glob_count > 0 + || (module_inst->export_global_count > 0 && !(module_inst->export_globals = export_globals_instantiate( - module, module_inst, module_inst->export_glob_count, + module, module_inst, module_inst->export_global_count, error_buf, error_buf_size))) #endif -#if WASM_ENABLE_FAST_JIT != 0 - || (module_inst->function_count > 0 +#if WASM_ENABLE_JIT != 0 + || (module_inst->e->function_count > 0 + && !init_func_ptrs(module_inst, module, error_buf, error_buf_size)) +#endif +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 + || (module_inst->e->function_count > 0 && !init_func_type_indexes(module_inst, error_buf, error_buf_size)) #endif ) { @@ -1437,7 +1540,7 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, if (global_count > 0) { /* Initialize the global data */ global_data = module_inst->global_data; - global_data_end = global_data + global_data_size; + global_data_end = global_data + module->global_data_size; global = globals; for (i = 0; i < global_count; i++, global++) { switch (global->type) { @@ -1457,6 +1560,13 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, &global->initial_value.i64, sizeof(int64)); global_data += sizeof(int64); break; +#if WASM_ENABLE_SIMD != 0 + case VALUE_TYPE_V128: + bh_memcpy_s(global_data, (uint32)sizeof(V128), + &global->initial_value.v128, sizeof(V128)); + global_data += sizeof(V128); + break; +#endif default: bh_assert(0); } @@ -1469,9 +1579,6 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, } /* Initialize the memory data with data segment section */ - module_inst->default_memory = - module_inst->memory_count ? module_inst->memories[0] : NULL; - for (i = 0; i < module->data_seg_count; i++) { WASMMemoryInstance *memory = NULL; uint8 *memory_data = NULL; @@ -1554,30 +1661,41 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, } /* Initialize the table data with table segment section */ - module_inst->default_table = - module_inst->table_count ? module_inst->tables[0] : NULL; - /* in case there is no table */ for (i = 0; module_inst->table_count > 0 && i < module->table_seg_count; i++) { WASMTableSeg *table_seg = module->table_segments + i; /* has check it in loader */ WASMTableInstance *table = module_inst->tables[table_seg->table_index]; + uint32 *table_data; +#if WASM_ENABLE_REF_TYPES != 0 + uint8 tbl_elem_type; + uint32 tbl_init_size, tbl_max_size; +#endif + bh_assert(table); #if WASM_ENABLE_REF_TYPES != 0 - if (table->elem_type != VALUE_TYPE_FUNCREF - && table->elem_type != VALUE_TYPE_EXTERNREF) { + (void)wasm_runtime_get_table_inst_elem_type( + (WASMModuleInstanceCommon *)module_inst, table_seg->table_index, + &tbl_elem_type, &tbl_init_size, &tbl_max_size); + if (tbl_elem_type != VALUE_TYPE_FUNCREF + && tbl_elem_type != VALUE_TYPE_EXTERNREF) { set_error_buf(error_buf, error_buf_size, "elements segment does not fit"); goto fail; } + (void)tbl_init_size; + (void)tbl_max_size; #endif - uint32 *table_data = (uint32 *)table->base_addr; + table_data = table->elems; #if WASM_ENABLE_MULTI_MODULE != 0 - table_data = table->table_inst_linked - ? (uint32 *)table->table_inst_linked->base_addr - : table_data; + if (table_seg->table_index < module->import_table_count + && module_inst->e->table_insts_linked[table_seg->table_index]) { + table_data = + module_inst->e->table_insts_linked[table_seg->table_index] + ->elems; + } #endif bh_assert(table_data); @@ -1586,7 +1704,6 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, continue; #endif - /* init vec(funcidx) or vec(expr) */ #if WASM_ENABLE_REF_TYPES != 0 bh_assert(table_seg->base_offset.init_expr_type == INIT_EXPR_TYPE_I32_CONST @@ -1603,6 +1720,7 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, == INIT_EXPR_TYPE_GET_GLOBAL); #endif + /* init vec(funcidx) or vec(expr) */ if (table_seg->base_offset.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) { if (!check_global_init_expr(module, @@ -1665,9 +1783,6 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, table_seg->func_indexes, (uint32)(length * sizeof(uint32))); } - /* module instance type */ - module_inst->module_type = Wasm_Module_Bytecode; - /* Initialize the thread related data */ if (stack_size == 0) stack_size = DEFAULT_WASM_STACK_SIZE; @@ -1678,18 +1793,18 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, module_inst->default_wasm_stack_size = stack_size; if (module->malloc_function != (uint32)-1) { - module_inst->malloc_function = - &module_inst->functions[module->malloc_function]; + module_inst->e->malloc_function = + &module_inst->e->functions[module->malloc_function]; } if (module->free_function != (uint32)-1) { - module_inst->free_function = - &module_inst->functions[module->free_function]; + module_inst->e->free_function = + &module_inst->e->functions[module->free_function]; } if (module->retain_function != (uint32)-1) { - module_inst->retain_function = - &module_inst->functions[module->retain_function]; + module_inst->e->retain_function = + &module_inst->e->functions[module->retain_function]; } #if WASM_ENABLE_LIBC_WASI != 0 @@ -1714,8 +1829,8 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, if (module->start_function != (uint32)-1) { /* TODO: fix start function can be import function issue */ if (module->start_function >= module->import_function_count) - module_inst->start_function = - &module_inst->functions[module->start_function]; + module_inst->e->start_function = + &module_inst->e->functions[module->start_function]; } /* Execute __post_instantiate function */ @@ -1763,7 +1878,12 @@ wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst) if (!module_inst) return; -#if WASM_ENABLE_FAST_JIT != 0 +#if WASM_ENABLE_JIT != 0 + if (module_inst->func_ptrs) + wasm_runtime_free(module_inst->func_ptrs); +#endif + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 if (module_inst->func_type_indexes) wasm_runtime_free(module_inst->func_type_indexes); #endif @@ -1790,18 +1910,15 @@ wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst) wasm_runtime_free(module_inst->import_func_ptrs); } - tables_deinstantiate(module_inst->tables, module_inst->table_count); - functions_deinstantiate(module_inst->functions, - module_inst->function_count); - globals_deinstantiate(module_inst->globals); + tables_deinstantiate(module_inst); + functions_deinstantiate(module_inst->e->functions, + module_inst->e->function_count); + globals_deinstantiate(module_inst->e->globals); export_functions_deinstantiate(module_inst->export_functions); #if WASM_ENABLE_MULTI_MODULE != 0 export_globals_deinstantiate(module_inst->export_globals); #endif - if (module_inst->global_data) - wasm_runtime_free(module_inst->global_data); - #if WASM_ENABLE_REF_TYPES != 0 wasm_externref_cleanup((WASMModuleInstanceCommon *)module_inst); #endif @@ -1825,6 +1942,11 @@ wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst) } #endif +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (module_inst->e->mem_lock_inited) + os_mutex_destroy(&module_inst->e->mem_lock); +#endif + wasm_runtime_free(module_inst); } @@ -1845,7 +1967,7 @@ WASMGlobalInstance * wasm_lookup_global(const WASMModuleInstance *module_inst, const char *name) { uint32 i; - for (i = 0; i < module_inst->export_glob_count; i++) + for (i = 0; i < module_inst->export_global_count; i++) if (!strcmp(module_inst->export_globals[i].name, name)) return module_inst->export_globals[i].global; return NULL; @@ -1894,113 +2016,6 @@ clear_wasi_proc_exit_exception(WASMModuleInstance *module_inst) #ifdef OS_ENABLE_HW_BOUND_CHECK -#ifndef BH_PLATFORM_WINDOWS -void -wasm_signal_handler(WASMSignalInfo *sig_info) -{ - WASMExecEnv *exec_env_tls = sig_info->exec_env_tls; - void *sig_addr = sig_info->sig_addr; - WASMModuleInstance *module_inst; - WASMMemoryInstance *memory_inst; - WASMJmpBuf *jmpbuf_node; - uint8 *mapped_mem_start_addr = NULL; - uint8 *mapped_mem_end_addr = NULL; - uint8 *stack_min_addr; - uint32 page_size; - uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; - - /* Check whether current thread is running wasm function */ - if (exec_env_tls && exec_env_tls->handle == os_self_thread() - && (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) { - /* Get mapped mem info of current instance */ - module_inst = (WASMModuleInstance *)exec_env_tls->module_inst; - /* Get the default memory instance */ - memory_inst = module_inst->default_memory; - if (memory_inst) { - mapped_mem_start_addr = (uint8 *)memory_inst->memory_data; - mapped_mem_end_addr = - (uint8 *)memory_inst->memory_data + 8 * (uint64)BH_GB; - } - - /* Get stack info of current thread */ - page_size = os_getpagesize(); - stack_min_addr = os_thread_get_stack_boundary(); - - if (memory_inst - && (mapped_mem_start_addr <= (uint8 *)sig_addr - && (uint8 *)sig_addr < mapped_mem_end_addr)) { - /* The address which causes segmentation fault is inside - the memory instance's guard regions */ - wasm_set_exception(module_inst, "out of bounds memory access"); - os_longjmp(jmpbuf_node->jmpbuf, 1); - } - else if (stack_min_addr - page_size <= (uint8 *)sig_addr - && (uint8 *)sig_addr - < stack_min_addr + page_size * guard_page_count) { - /* The address which causes segmentation fault is inside - native thread's guard page */ - wasm_set_exception(module_inst, "native stack overflow"); - os_longjmp(jmpbuf_node->jmpbuf, 1); - } - } -} -#else /* else of BH_PLATFORM_WINDOWS */ -LONG -wasm_exception_handler(WASMSignalInfo *sig_info) -{ - WASMExecEnv *exec_env_tls = sig_info->exec_env_tls; - EXCEPTION_POINTERS *exce_info = sig_info->exce_info; - PEXCEPTION_RECORD ExceptionRecord = exce_info->ExceptionRecord; - uint8 *sig_addr = (uint8 *)ExceptionRecord->ExceptionInformation[1]; - WASMModuleInstance *module_inst; - WASMMemoryInstance *memory_inst; - WASMJmpBuf *jmpbuf_node; - uint8 *mapped_mem_start_addr = NULL; - uint8 *mapped_mem_end_addr = NULL; - uint32 page_size = os_getpagesize(); - - if (exec_env_tls && exec_env_tls->handle == os_self_thread() - && (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) { - module_inst = (WASMModuleInstance *)exec_env_tls->module_inst; - if (ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { - /* Get the default memory instance */ - memory_inst = module_inst->default_memory; - if (memory_inst) { - mapped_mem_start_addr = (uint8 *)memory_inst->memory_data; - mapped_mem_end_addr = - (uint8 *)memory_inst->memory_data + 8 * (uint64)BH_GB; - if (mapped_mem_start_addr <= (uint8 *)sig_addr - && (uint8 *)sig_addr < mapped_mem_end_addr) { - /* The address which causes segmentation fault is inside - the memory instance's guard regions. - Set exception and let the wasm func continue to run, when - the wasm func returns, the caller will check whether the - exception is thrown and return to runtime. */ - wasm_set_exception(module_inst, - "out of bounds memory access"); - /* Skip current instruction */ - return EXCEPTION_CONTINUE_SEARCH; - } - } - } - else if (ExceptionRecord->ExceptionCode == EXCEPTION_STACK_OVERFLOW) { - /* Set stack overflow exception and let the wasm func continue - to run, when the wasm func returns, the caller will check - whether the exception is thrown and return to runtime, and - the damaged stack will be recovered by _resetstkoflw(). */ - wasm_set_exception(module_inst, "native stack overflow"); - return EXCEPTION_CONTINUE_SEARCH; - } - } - - os_printf("Unhandled exception thrown: exception code: 0x%lx, " - "exception address: %p, exception information: %p\n", - ExceptionRecord->ExceptionCode, ExceptionRecord->ExceptionAddress, - sig_addr); - return EXCEPTION_CONTINUE_SEARCH; -} -#endif /* end of BH_PLATFORM_WINDOWS */ - static void call_wasm_with_hw_bound_check(WASMModuleInstance *module_inst, WASMExecEnv *exec_env, @@ -2138,42 +2153,6 @@ wasm_create_exec_env_and_call_function(WASMModuleInstance *module_inst, return ret; } -bool -wasm_create_exec_env_singleton(WASMModuleInstance *module_inst) -{ - WASMExecEnv *exec_env = NULL; - - if (module_inst->exec_env_singleton) { - return true; - } - - exec_env = wasm_exec_env_create((WASMModuleInstanceCommon *)module_inst, - module_inst->default_wasm_stack_size); - if (exec_env) - module_inst->exec_env_singleton = exec_env; - - return exec_env ? true : false; -} - -void -wasm_set_exception(WASMModuleInstance *module_inst, const char *exception) -{ - if (exception) - snprintf(module_inst->cur_exception, sizeof(module_inst->cur_exception), - "Exception: %s", exception); - else - module_inst->cur_exception[0] = '\0'; -} - -const char * -wasm_get_exception(WASMModuleInstance *module_inst) -{ - if (module_inst->cur_exception[0] == '\0') - return NULL; - else - return module_inst->cur_exception; -} - #if WASM_ENABLE_PERF_PROFILING != 0 void wasm_dump_perf_profiling(const WASMModuleInstance *module_inst) @@ -2184,8 +2163,8 @@ wasm_dump_perf_profiling(const WASMModuleInstance *module_inst) uint32 i, j; os_printf("Performance profiler data:\n"); - for (i = 0; i < module_inst->function_count; i++) { - func_inst = module_inst->functions + i; + for (i = 0; i < module_inst->e->function_count; i++) { + func_inst = module_inst->e->functions + i; if (func_inst->is_import_func) { func_name = func_inst->u.func_import->field_name; } @@ -2209,13 +2188,13 @@ wasm_dump_perf_profiling(const WASMModuleInstance *module_inst) os_printf(" func %s, execution time: %.3f ms, execution count: %d " "times\n", func_name, - module_inst->functions[i].total_exec_time / 1000.0f, - module_inst->functions[i].total_exec_cnt); + module_inst->e->functions[i].total_exec_time / 1000.0f, + module_inst->e->functions[i].total_exec_cnt); else os_printf(" func %d, execution time: %.3f ms, execution count: %d " "times\n", - i, module_inst->functions[i].total_exec_time / 1000.0f, - module_inst->functions[i].total_exec_cnt); + i, module_inst->e->functions[i].total_exec_time / 1000.0f, + module_inst->e->functions[i].total_exec_cnt); } } #endif @@ -2224,7 +2203,7 @@ uint32 wasm_module_malloc(WASMModuleInstance *module_inst, uint32 size, void **p_native_addr) { - WASMMemoryInstance *memory = module_inst->default_memory; + WASMMemoryInstance *memory = wasm_get_default_memory(module_inst); uint8 *addr = NULL; uint32 offset = 0; @@ -2236,15 +2215,15 @@ wasm_module_malloc(WASMModuleInstance *module_inst, uint32 size, if (memory->heap_handle) { addr = mem_allocator_malloc(memory->heap_handle, size); } - else if (module_inst->malloc_function && module_inst->free_function) { - if (!execute_malloc_function(module_inst, module_inst->malloc_function, - module_inst->retain_function, size, - &offset)) { + else if (module_inst->e->malloc_function && module_inst->e->free_function) { + if (!execute_malloc_function( + module_inst, module_inst->e->malloc_function, + module_inst->e->retain_function, size, &offset)) { return 0; } /* If we use app's malloc function, the default memory may be changed while memory growing */ - memory = module_inst->default_memory; + memory = wasm_get_default_memory(module_inst); addr = offset ? memory->memory_data + offset : NULL; } @@ -2269,7 +2248,7 @@ uint32 wasm_module_realloc(WASMModuleInstance *module_inst, uint32 ptr, uint32 size, void **p_native_addr) { - WASMMemoryInstance *memory = module_inst->default_memory; + WASMMemoryInstance *memory = wasm_get_default_memory(module_inst); uint8 *addr = NULL; if (!memory) { @@ -2304,7 +2283,7 @@ void wasm_module_free(WASMModuleInstance *module_inst, uint32 ptr) { if (ptr) { - WASMMemoryInstance *memory = module_inst->default_memory; + WASMMemoryInstance *memory = wasm_get_default_memory(module_inst); uint8 *addr; if (!memory) { @@ -2317,10 +2296,11 @@ wasm_module_free(WASMModuleInstance *module_inst, uint32 ptr) && addr < memory->heap_data_end) { mem_allocator_free(memory->heap_handle, addr); } - else if (module_inst->malloc_function && module_inst->free_function - && memory->memory_data <= addr + else if (module_inst->e->malloc_function + && module_inst->e->free_function && memory->memory_data <= addr && addr < memory->memory_data_end) { - execute_free_function(module_inst, module_inst->free_function, ptr); + execute_free_function(module_inst, module_inst->e->free_function, + ptr); } } } @@ -2333,338 +2313,22 @@ wasm_module_dup_data(WASMModuleInstance *module_inst, const char *src, uint32 buffer_offset = wasm_module_malloc(module_inst, size, (void **)&buffer); if (buffer_offset != 0) { - buffer = wasm_addr_app_to_native(module_inst, buffer_offset); + buffer = wasm_runtime_addr_app_to_native( + (WASMModuleInstanceCommon *)module_inst, buffer_offset); bh_memcpy_s(buffer, size, src, size); } return buffer_offset; } -bool -wasm_validate_app_addr(WASMModuleInstance *module_inst, uint32 app_offset, - uint32 size) -{ - WASMMemoryInstance *memory = module_inst->default_memory; - uint32 memory_data_size; - - if (!memory) { - goto fail; - } - - memory_data_size = memory->num_bytes_per_page * memory->cur_page_count; - - /* integer overflow check */ - if (app_offset > UINT32_MAX - size) { - goto fail; - } - - if (app_offset + size <= memory_data_size) { - return true; - } -fail: - wasm_set_exception(module_inst, "out of bounds memory access"); - return false; -} - -bool -wasm_validate_native_addr(WASMModuleInstance *module_inst, void *native_ptr, - uint32 size) -{ - WASMMemoryInstance *memory = module_inst->default_memory; - uint8 *addr = (uint8 *)native_ptr; - - if (!memory) { - goto fail; - } - - /* integer overflow check */ - if ((uintptr_t)addr > UINTPTR_MAX - size) { - goto fail; - } - - if (memory->memory_data <= addr && addr + size <= memory->memory_data_end) { - return true; - } -fail: - wasm_set_exception(module_inst, "out of bounds memory access"); - return false; -} - -void * -wasm_addr_app_to_native(WASMModuleInstance *module_inst, uint32 app_offset) -{ - WASMMemoryInstance *memory = module_inst->default_memory; - uint8 *addr; - - if (!memory) - return NULL; - - addr = memory->memory_data + app_offset; - - if (memory->memory_data <= addr && addr < memory->memory_data_end) - return addr; - return NULL; -} - -uint32 -wasm_addr_native_to_app(WASMModuleInstance *module_inst, void *native_ptr) -{ - WASMMemoryInstance *memory = module_inst->default_memory; - uint8 *addr = (uint8 *)native_ptr; - - if (!memory) - return 0; - - if (memory->memory_data <= addr && addr < memory->memory_data_end) - return (uint32)(addr - memory->memory_data); - return 0; -} - -bool -wasm_get_app_addr_range(WASMModuleInstance *module_inst, uint32 app_offset, - uint32 *p_app_start_offset, uint32 *p_app_end_offset) -{ - WASMMemoryInstance *memory = module_inst->default_memory; - uint32 memory_data_size; - - if (!memory) - return false; - - memory_data_size = memory->num_bytes_per_page * memory->cur_page_count; - - if (app_offset < memory_data_size) { - if (p_app_start_offset) - *p_app_start_offset = 0; - if (p_app_end_offset) - *p_app_end_offset = memory_data_size; - return true; - } - return false; -} - -bool -wasm_get_native_addr_range(WASMModuleInstance *module_inst, uint8 *native_ptr, - uint8 **p_native_start_addr, - uint8 **p_native_end_addr) -{ - WASMMemoryInstance *memory = module_inst->default_memory; - uint8 *addr = (uint8 *)native_ptr; - - if (!memory) - return false; - - if (memory->memory_data <= addr && addr < memory->memory_data_end) { - if (p_native_start_addr) - *p_native_start_addr = memory->memory_data; - if (p_native_end_addr) - *p_native_end_addr = memory->memory_data_end; - return true; - } - return false; -} - -#ifndef OS_ENABLE_HW_BOUND_CHECK -bool -wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) -{ - WASMMemoryInstance *memory = module->default_memory; - uint8 *memory_data_old, *memory_data_new, *heap_data_old; - uint32 num_bytes_per_page, heap_size, total_size_old; - uint32 cur_page_count, max_page_count, total_page_count; - uint64 total_size_new; - bool ret = true; - - if (!memory) - return false; - - heap_data_old = memory->heap_data; - heap_size = (uint32)(memory->heap_data_end - memory->heap_data); - - memory_data_old = memory->memory_data; - total_size_old = memory->memory_data_size; - - num_bytes_per_page = memory->num_bytes_per_page; - cur_page_count = memory->cur_page_count; - max_page_count = memory->max_page_count; - total_page_count = inc_page_count + cur_page_count; - total_size_new = num_bytes_per_page * (uint64)total_page_count; - - if (inc_page_count <= 0) - /* No need to enlarge memory */ - return true; - - if (total_page_count < cur_page_count /* integer overflow */ - || total_page_count > max_page_count) { - return false; - } - - bh_assert(total_size_new <= 4 * (uint64)BH_GB); - if (total_size_new > UINT32_MAX) { - /* Resize to 1 page with size 4G-1 */ - num_bytes_per_page = UINT32_MAX; - total_page_count = max_page_count = 1; - total_size_new = UINT32_MAX; - } - -#if WASM_ENABLE_SHARED_MEMORY != 0 - if (memory->is_shared) { - memory->num_bytes_per_page = num_bytes_per_page; - memory->cur_page_count = total_page_count; - memory->max_page_count = max_page_count; - /* No need to update memory->memory_data_size as it is - initialized with the maximum memory data size for - shared memory */ - return true; - } -#endif - - if (heap_size > 0) { - if (mem_allocator_is_heap_corrupted(memory->heap_handle)) { - wasm_runtime_show_app_heap_corrupted_prompt(); - return false; - } - } - - if (!(memory_data_new = - wasm_runtime_realloc(memory_data_old, (uint32)total_size_new))) { - if (!(memory_data_new = wasm_runtime_malloc((uint32)total_size_new))) { - return false; - } - if (memory_data_old) { - bh_memcpy_s(memory_data_new, (uint32)total_size_new, - memory_data_old, total_size_old); - wasm_runtime_free(memory_data_old); - } - } - - memset(memory_data_new + total_size_old, 0, - (uint32)total_size_new - total_size_old); - - if (heap_size > 0) { - if (mem_allocator_migrate(memory->heap_handle, - (char *)heap_data_old - + (memory_data_new - memory_data_old), - heap_size) - != 0) { - /* Don't return here as memory->memory_data is obsolete and - must be updated to be correctly used later. */ - ret = false; - } - } - - memory->heap_data = memory_data_new + (heap_data_old - memory_data_old); - memory->heap_data_end = memory->heap_data + heap_size; - - memory->num_bytes_per_page = num_bytes_per_page; - memory->cur_page_count = total_page_count; - memory->max_page_count = max_page_count; - memory->memory_data_size = (uint32)total_size_new; - - memory->memory_data = memory_data_new; - memory->memory_data_end = memory_data_new + (uint32)total_size_new; - -#if WASM_ENABLE_FAST_JIT != 0 -#if UINTPTR_MAX == UINT64_MAX - memory->mem_bound_check_1byte = total_size_new - 1; - memory->mem_bound_check_2bytes = total_size_new - 2; - memory->mem_bound_check_4bytes = total_size_new - 4; - memory->mem_bound_check_8bytes = total_size_new - 8; - memory->mem_bound_check_16bytes = total_size_new - 16; -#else - memory->mem_bound_check_1byte = (uint32)total_size_new - 1; - memory->mem_bound_check_2bytes = (uint32)total_size_new - 2; - memory->mem_bound_check_4bytes = (uint32)total_size_new - 4; - memory->mem_bound_check_8bytes = (uint32)total_size_new - 8; - memory->mem_bound_check_16bytes = (uint32)total_size_new - 16; -#endif -#endif - - return ret; -} -#else -bool -wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) -{ - WASMMemoryInstance *memory = module->default_memory; - uint32 num_bytes_per_page, total_size_old; - uint32 cur_page_count, max_page_count, total_page_count; - uint64 total_size_new; - - if (!memory) - return false; - - num_bytes_per_page = memory->num_bytes_per_page; - cur_page_count = memory->cur_page_count; - max_page_count = memory->max_page_count; - total_size_old = num_bytes_per_page * cur_page_count; - total_page_count = inc_page_count + cur_page_count; - total_size_new = num_bytes_per_page * (uint64)total_page_count; - - if (inc_page_count <= 0) - /* No need to enlarge memory */ - return true; - - if (total_page_count < cur_page_count /* integer overflow */ - || total_page_count > max_page_count) { - return false; - } - - bh_assert(total_size_new <= 4 * (uint64)BH_GB); - if (total_size_new > UINT32_MAX) { - /* Resize to 1 page with size 4G-1 */ - num_bytes_per_page = UINT32_MAX; - total_page_count = max_page_count = 1; - total_size_new = UINT32_MAX; - } - -#ifdef BH_PLATFORM_WINDOWS - if (!os_mem_commit(memory->memory_data_end, - (uint32)total_size_new - total_size_old, - MMAP_PROT_READ | MMAP_PROT_WRITE)) { - return false; - } -#endif - - if (os_mprotect(memory->memory_data_end, - (uint32)total_size_new - total_size_old, - MMAP_PROT_READ | MMAP_PROT_WRITE) - != 0) { -#ifdef BH_PLATFORM_WINDOWS - os_mem_decommit(memory->memory_data_end, - (uint32)total_size_new - total_size_old); -#endif - return false; - } - - /* The increased pages are filled with zero by the OS when os_mmap, - no need to memset it again here */ - - memory->num_bytes_per_page = num_bytes_per_page; - memory->cur_page_count = total_page_count; - memory->max_page_count = max_page_count; - memory->memory_data_size = (uint32)total_size_new; - memory->memory_data_end = memory->memory_data + (uint32)total_size_new; - -#if WASM_ENABLE_FAST_JIT != 0 - memory->mem_bound_check_1byte = total_size_new - 1; - memory->mem_bound_check_2bytes = total_size_new - 2; - memory->mem_bound_check_4bytes = total_size_new - 4; - memory->mem_bound_check_8bytes = total_size_new - 8; - memory->mem_bound_check_16bytes = total_size_new - 16; -#endif - - return true; -} -#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ - #if WASM_ENABLE_REF_TYPES != 0 bool wasm_enlarge_table(WASMModuleInstance *module_inst, uint32 table_idx, - uint32 inc_entries, uint32 init_val) + uint32 inc_size, uint32 init_val) { - uint32 entry_count, *new_table_data_start, i; + uint32 total_size, *new_table_data_start, i; WASMTableInstance *table_inst; - if (!inc_entries) { + if (!inc_size) { return true; } @@ -2674,24 +2338,22 @@ wasm_enlarge_table(WASMModuleInstance *module_inst, uint32 table_idx, return false; } - if (inc_entries > UINT32_MAX - table_inst->cur_size) { + if (inc_size > UINT32_MAX - table_inst->cur_size) { return false; } - entry_count = table_inst->cur_size + inc_entries; - if (entry_count > table_inst->max_size) { + total_size = table_inst->cur_size + inc_size; + if (total_size > table_inst->max_size) { return false; } /* fill in */ - new_table_data_start = - (uint32 *)((uint8 *)table_inst + offsetof(WASMTableInstance, base_addr)) - + table_inst->cur_size; - for (i = 0; i < inc_entries; ++i) { + new_table_data_start = table_inst->elems + table_inst->cur_size; + for (i = 0; i < inc_size; ++i) { new_table_data_start[i] = init_val; } - table_inst->cur_size = entry_count; + table_inst->cur_size = total_size; return true; } #endif /* WASM_ENABLE_REF_TYPES != 0 */ @@ -2719,11 +2381,7 @@ call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, goto got_exception; } - /** - * please be aware that table_inst->base_addr may point - * to another module's table - **/ - func_idx = ((uint32 *)table_inst->base_addr)[elem_idx]; + func_idx = table_inst->elems[elem_idx]; if (func_idx == NULL_REF) { wasm_set_exception(module_inst, "uninitialized element"); goto got_exception; @@ -2732,12 +2390,12 @@ call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, /** * we insist to call functions owned by the module itself **/ - if (func_idx >= module_inst->function_count) { + if (func_idx >= module_inst->e->function_count) { wasm_set_exception(module_inst, "unknown function"); goto got_exception; } - func_inst = module_inst->functions + func_idx; + func_inst = module_inst->e->functions + func_idx; if (check_type_idx) { WASMType *cur_type = module_inst->module->types[type_idx]; @@ -2770,16 +2428,6 @@ wasm_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, return call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, false, 0); } -#if WASM_ENABLE_FAST_JIT != 0 -bool -jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, - uint32 type_idx, uint32 argc, uint32 argv[]) -{ - return call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, true, - type_idx); -} -#endif - #if WASM_ENABLE_THREAD_MGR != 0 bool wasm_set_aux_stack(WASMExecEnv *exec_env, uint32 start_offset, uint32 size) @@ -2799,8 +2447,9 @@ wasm_set_aux_stack(WASMExecEnv *exec_env, uint32 start_offset, uint32 size) if (stack_top_idx != (uint32)-1) { /* The aux stack top is a wasm global, set the initial value for the global */ - uint8 *global_addr = module_inst->global_data - + module_inst->globals[stack_top_idx].data_offset; + uint8 *global_addr = + module_inst->global_data + + module_inst->e->globals[stack_top_idx].data_offset; *(int32 *)global_addr = start_offset; /* The aux stack boundary is a constant value, set the value to exec_env */ @@ -2917,48 +2566,34 @@ wasm_get_module_inst_mem_consumption(const WASMModuleInstance *module_inst, memset(mem_conspn, 0, sizeof(*mem_conspn)); - mem_conspn->module_inst_struct_size = sizeof(WASMModuleInstance); + mem_conspn->module_inst_struct_size = (uint8 *)module_inst->e + - (uint8 *)module_inst + + sizeof(WASMModuleInstanceExtra); mem_conspn->memories_size = sizeof(WASMMemoryInstance *) * module_inst->memory_count; for (i = 0; i < module_inst->memory_count; i++) { WASMMemoryInstance *memory = module_inst->memories[i]; - size = sizeof(WASMMemoryInstance) - + memory->num_bytes_per_page * memory->cur_page_count; + size = memory->num_bytes_per_page * memory->cur_page_count; mem_conspn->memories_size += size; mem_conspn->app_heap_size += memory->heap_data_end - memory->heap_data; /* size of app heap structure */ mem_conspn->memories_size += mem_allocator_get_heap_struct_size(); + /* Module instance structures have been appened into the end of + module instance */ } mem_conspn->tables_size = sizeof(WASMTableInstance *) * module_inst->table_count; - for (i = 0; i < module_inst->table_count; i++) { - WASMTableInstance *table = module_inst->tables[i]; -#if WASM_ENABLE_MULTI_MODULE != 0 - if (table->table_inst_linked) { - size = offsetof(WASMTableInstance, base_addr); - } - else -#endif - { - size = offsetof(WASMTableInstance, base_addr) - + sizeof(uint32) * table->cur_size; - } - mem_conspn->tables_size += size; - } + /* Table instance structures and table elements have been appened into + the end of module instance */ mem_conspn->functions_size = - sizeof(WASMFunctionInstance) * module_inst->function_count; + sizeof(WASMFunctionInstance) * module_inst->e->function_count; mem_conspn->globals_size = - sizeof(WASMGlobalInstance) * module_inst->global_count; - if (module_inst->global_count > 0) { - WASMGlobalInstance *global = - &module_inst->globals[module_inst->global_count - 1]; - mem_conspn->globals_size += - global->data_offset + wasm_value_type_size(global->type); - } + sizeof(WASMGlobalInstance) * module_inst->e->global_count; + /* Global data has been appened into the end of module instance */ mem_conspn->exports_size = sizeof(WASMExportFuncInstance) * module_inst->export_func_count; @@ -3016,7 +2651,7 @@ wasm_interp_create_call_stack(struct WASMExecEnv *exec_env) /* place holder, will overwrite it in wasm_c_api */ frame.instance = module_inst; frame.module_offset = 0; - frame.func_index = (uint32)(func_inst - module_inst->functions); + frame.func_index = (uint32)(func_inst - module_inst->e->functions); func_code_base = wasm_get_func_code(func_inst); if (!cur_frame->ip || !func_code_base) { @@ -3132,3 +2767,396 @@ wasm_interp_dump_call_stack(struct WASMExecEnv *exec_env, bool print, char *buf, return total_len + 1; } #endif /* end of WASM_ENABLE_DUMP_CALL_STACK */ + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 +void +jit_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id) +{ + wasm_set_exception_with_id(module_inst, id); +} + +bool +jit_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, + uint32 app_buf_addr, uint32 app_buf_size, + void **p_native_addr) +{ + return wasm_check_app_addr_and_convert(module_inst, is_str, app_buf_addr, + app_buf_size, p_native_addr); +} +#endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 */ + +#if WASM_ENABLE_FAST_JIT != 0 +bool +fast_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 type_idx, uint32 argc, uint32 *argv) +{ + return call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, true, + type_idx); +} +#endif + +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + +bool +llvm_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 argc, uint32 *argv) +{ +#if WASM_ENABLE_JIT != 0 + if (Wasm_Module_AoT == exec_env->module_inst->module_type) { + return aot_call_indirect(exec_env, tbl_idx, elem_idx, argc, argv); + } +#endif + return call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, false, 0); +} + +bool +llvm_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc, + uint32 *argv) +{ + WASMModuleInstance *module_inst; + WASMModule *module; + uint32 *func_type_indexes; + uint32 func_type_idx; + WASMType *func_type; + void *func_ptr; + WASMFunctionImport *import_func; + const char *signature; + void *attachment; + char buf[96]; + +#if WASM_ENABLE_JIT != 0 + if (Wasm_Module_AoT == exec_env->module_inst->module_type) { + return aot_invoke_native(exec_env, func_idx, argc, argv); + } +#endif + + module_inst = (WASMModuleInstance *)wasm_runtime_get_module_inst(exec_env); + module = module_inst->module; + func_type_indexes = module_inst->func_type_indexes; + func_type_idx = func_type_indexes[func_idx]; + func_type = module->types[func_type_idx]; + func_ptr = module_inst->func_ptrs[func_idx]; + + bh_assert(func_idx < module->import_function_count); + + import_func = &module->import_functions[func_idx].u.function; + if (!func_ptr) { + snprintf(buf, sizeof(buf), + "failed to call unlinked import function (%s, %s)", + import_func->module_name, import_func->field_name); + wasm_set_exception(module_inst, buf); + return false; + } + + attachment = import_func->attachment; + if (import_func->call_conv_wasm_c_api) { + return wasm_runtime_invoke_c_api_native( + (WASMModuleInstanceCommon *)module_inst, func_ptr, func_type, argc, + argv, import_func->wasm_c_api_with_env, attachment); + } + else if (!import_func->call_conv_raw) { + signature = import_func->signature; + return wasm_runtime_invoke_native(exec_env, func_ptr, func_type, + signature, attachment, argv, argc, + argv); + } + else { + signature = import_func->signature; + return wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type, + signature, attachment, argv, argc, + argv); + } +} + +#if WASM_ENABLE_BULK_MEMORY != 0 +bool +llvm_jit_memory_init(WASMModuleInstance *module_inst, uint32 seg_index, + uint32 offset, uint32 len, uint32 dst) +{ + WASMMemoryInstance *memory_inst; + WASMModule *module; + uint8 *data = NULL; + uint8 *maddr; + uint64 seg_len = 0; + +#if WASM_ENABLE_JIT != 0 + if (Wasm_Module_AoT == module_inst->module_type) { + return aot_memory_init(module_inst, seg_index, offset, len, dst); + } +#endif + + memory_inst = wasm_get_default_memory(module_inst); + module = module_inst->module; + seg_len = module->data_segments[seg_index]->data_length; + data = module->data_segments[seg_index]->data; + + if (!wasm_runtime_validate_app_addr((WASMModuleInstanceCommon *)module_inst, + dst, len)) + return false; + + if ((uint64)offset + (uint64)len > seg_len) { + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; + } + + maddr = wasm_runtime_addr_app_to_native( + (WASMModuleInstanceCommon *)module_inst, dst); + + bh_memcpy_s(maddr, memory_inst->memory_data_size - dst, data + offset, len); + return true; +} + +bool +llvm_jit_data_drop(WASMModuleInstance *module_inst, uint32 seg_index) +{ +#if WASM_ENABLE_JIT != 0 + if (Wasm_Module_AoT == module_inst->module_type) { + return aot_data_drop(module_inst, seg_index); + } +#endif + + module_inst->module->data_segments[seg_index]->data_length = 0; + /* Currently we can't free the dropped data segment + as they are stored in wasm bytecode */ + return true; +} +#endif /* end of WASM_ENABLE_BULK_MEMORY != 0 */ + +#if WASM_ENABLE_REF_TYPES != 0 +void +llvm_jit_drop_table_seg(WASMModuleInstance *module_inst, uint32 tbl_seg_idx) +{ + WASMTableSeg *tbl_segs; + +#if WASM_ENABLE_JIT != 0 + if (Wasm_Module_AoT == module_inst->module_type) { + return aot_drop_table_seg(module_inst, tbl_seg_idx); + } +#endif + + tbl_segs = module_inst->module->table_segments; + tbl_segs[tbl_seg_idx].is_dropped = true; +} + +void +llvm_jit_table_init(WASMModuleInstance *module_inst, uint32 tbl_idx, + uint32 tbl_seg_idx, uint32 length, uint32 src_offset, + uint32 dst_offset) +{ + WASMTableInstance *tbl_inst; + WASMTableSeg *tbl_seg; + +#if WASM_ENABLE_JIT != 0 + if (Wasm_Module_AoT == module_inst->module_type) { + return aot_table_init(module_inst, tbl_idx, tbl_seg_idx, length, + src_offset, dst_offset); + } +#endif + + tbl_inst = wasm_get_table_inst(module_inst, tbl_idx); + tbl_seg = module_inst->module->table_segments + tbl_seg_idx; + + bh_assert(tbl_inst); + bh_assert(tbl_seg); + + if (!length) { + return; + } + + if (length + src_offset > tbl_seg->function_count + || dst_offset + length > tbl_inst->cur_size) { + jit_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS); + return; + } + + if (tbl_seg->is_dropped) { + jit_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS); + return; + } + + if (!wasm_elem_is_passive(tbl_seg->mode)) { + jit_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS); + return; + } + + bh_memcpy_s((uint8 *)tbl_inst + offsetof(WASMTableInstance, elems) + + dst_offset * sizeof(uint32), + (uint32)sizeof(uint32) * (tbl_inst->cur_size - dst_offset), + tbl_seg->func_indexes + src_offset, + (uint32)(length * sizeof(uint32))); +} + +void +llvm_jit_table_copy(WASMModuleInstance *module_inst, uint32 src_tbl_idx, + uint32 dst_tbl_idx, uint32 length, uint32 src_offset, + uint32 dst_offset) +{ + WASMTableInstance *src_tbl_inst; + WASMTableInstance *dst_tbl_inst; + +#if WASM_ENABLE_JIT != 0 + if (Wasm_Module_AoT == module_inst->module_type) { + aot_table_copy(module_inst, src_tbl_idx, dst_tbl_idx, length, + src_offset, dst_offset); + return; + } +#endif + + src_tbl_inst = wasm_get_table_inst(module_inst, src_tbl_idx); + dst_tbl_inst = wasm_get_table_inst(module_inst, dst_tbl_idx); + bh_assert(src_tbl_inst); + bh_assert(dst_tbl_inst); + + if ((uint64)dst_offset + length > dst_tbl_inst->cur_size + || (uint64)src_offset + length > src_tbl_inst->cur_size) { + jit_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS); + return; + } + + /* if src_offset >= dst_offset, copy from front to back */ + /* if src_offset < dst_offset, copy from back to front */ + /* merge all together */ + bh_memmove_s((uint8 *)dst_tbl_inst + offsetof(WASMTableInstance, elems) + + sizeof(uint32) * dst_offset, + (uint32)sizeof(uint32) * (dst_tbl_inst->cur_size - dst_offset), + (uint8 *)src_tbl_inst + offsetof(WASMTableInstance, elems) + + sizeof(uint32) * src_offset, + (uint32)sizeof(uint32) * length); +} + +void +llvm_jit_table_fill(WASMModuleInstance *module_inst, uint32 tbl_idx, + uint32 length, uint32 val, uint32 data_offset) +{ + WASMTableInstance *tbl_inst; + +#if WASM_ENABLE_JIT != 0 + if (Wasm_Module_AoT == module_inst->module_type) { + aot_table_fill(module_inst, tbl_idx, length, val, data_offset); + return; + } +#endif + + tbl_inst = wasm_get_table_inst(module_inst, tbl_idx); + bh_assert(tbl_inst); + + if (data_offset + length > tbl_inst->cur_size) { + jit_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS); + return; + } + + for (; length != 0; data_offset++, length--) { + tbl_inst->elems[data_offset] = val; + } +} + +uint32 +llvm_jit_table_grow(WASMModuleInstance *module_inst, uint32 tbl_idx, + uint32 inc_size, uint32 init_val) +{ + WASMTableInstance *tbl_inst; + uint32 i, orig_size, total_size; + +#if WASM_ENABLE_JIT != 0 + if (Wasm_Module_AoT == module_inst->module_type) { + return aot_table_grow(module_inst, tbl_idx, inc_size, init_val); + } +#endif + + tbl_inst = wasm_get_table_inst(module_inst, tbl_idx); + if (!tbl_inst) { + return (uint32)-1; + } + + orig_size = tbl_inst->cur_size; + + if (!inc_size) { + return orig_size; + } + + if (tbl_inst->cur_size > UINT32_MAX - inc_size) { /* integer overflow */ + return (uint32)-1; + } + + total_size = tbl_inst->cur_size + inc_size; + if (total_size > tbl_inst->max_size) { + return (uint32)-1; + } + + /* fill in */ + for (i = 0; i < inc_size; ++i) { + tbl_inst->elems[tbl_inst->cur_size + i] = init_val; + } + + tbl_inst->cur_size = total_size; + return orig_size; +} +#endif /* end of WASM_ENABLE_REF_TYPES != 0 */ + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 || WASM_ENABLE_PERF_PROFILING != 0 +bool +llvm_jit_alloc_frame(WASMExecEnv *exec_env, uint32 func_index) +{ + WASMModuleInstance *module_inst; + WASMInterpFrame *frame; + uint32 size; + +#if WASM_ENABLE_JIT != 0 + if (Wasm_Module_AoT == exec_env->module_inst->module_type) { + return aot_alloc_frame(exec_env, func_index); + } +#endif + + module_inst = (WASMModuleInstance *)exec_env->module_inst; + size = wasm_interp_interp_frame_size(0); + + frame = wasm_exec_env_alloc_wasm_frame(exec_env, size); + if (!frame) { + wasm_set_exception(module_inst, "wasm operand stack overflow"); + return false; + } + + frame->function = module_inst->e->functions + func_index; + frame->ip = NULL; + frame->sp = frame->lp; +#if WASM_ENABLE_PERF_PROFILING != 0 + frame->time_started = os_time_get_boot_microsecond(); +#endif + frame->prev_frame = wasm_exec_env_get_cur_frame(exec_env); + wasm_exec_env_set_cur_frame(exec_env, frame); + + return true; +} + +void +llvm_jit_free_frame(WASMExecEnv *exec_env) +{ + WASMInterpFrame *frame; + WASMInterpFrame *prev_frame; + +#if WASM_ENABLE_JIT != 0 + if (Wasm_Module_AoT == exec_env->module_inst->module_type) { + aot_free_frame(exec_env); + return; + } +#endif + + frame = wasm_exec_env_get_cur_frame(exec_env); + prev_frame = frame->prev_frame; + +#if WASM_ENABLE_PERF_PROFILING != 0 + if (frame->function) { + frame->function->total_exec_time += + os_time_get_boot_microsecond() - frame->time_started; + frame->function->total_exec_cnt++; + } +#endif + wasm_exec_env_free_wasm_frame(exec_env, frame); + wasm_exec_env_set_cur_frame(exec_env, prev_frame); +} +#endif /* end of WASM_ENABLE_DUMP_CALL_STACK != 0 \ + || WASM_ENABLE_PERF_PROFILING != 0 */ + +#endif /* end of WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 */ diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 4cdbf5ba..3146b18f 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -21,6 +21,53 @@ typedef struct WASMMemoryInstance WASMMemoryInstance; typedef struct WASMTableInstance WASMTableInstance; typedef struct WASMGlobalInstance WASMGlobalInstance; +/** + * When LLVM JIT, WAMR compiler or AOT is enabled, we should ensure that + * some offsets of the same field in the interpreter module instance and + * aot module instance are the same, so that the LLVM JITed/AOTed code + * can smoothly access the interpreter module instance. + * Same for the memory instance and table instance. + * We use the macro DefPointer to define some related pointer fields. + */ +#if (WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 \ + || WASM_ENABLE_AOT != 0) \ + && UINTPTR_MAX == UINT32_MAX +/* Add u32 padding if LLVM JIT, WAMR compiler or AOT is enabled on + 32-bit platform */ +#define DefPointer(type, field) \ + type field; \ + uint32 field##_padding +#else +#define DefPointer(type, field) type field +#endif + +typedef enum WASMExceptionID { + EXCE_UNREACHABLE = 0, + EXCE_OUT_OF_MEMORY, + EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, + EXCE_INTEGER_OVERFLOW, + EXCE_INTEGER_DIVIDE_BY_ZERO, + EXCE_INVALID_CONVERSION_TO_INTEGER, + EXCE_INVALID_FUNCTION_TYPE_INDEX, + EXCE_INVALID_FUNCTION_INDEX, + EXCE_UNDEFINED_ELEMENT, + EXCE_UNINITIALIZED_ELEMENT, + EXCE_CALL_UNLINKED_IMPORT_FUNC, + EXCE_NATIVE_STACK_OVERFLOW, + EXCE_UNALIGNED_ATOMIC, + EXCE_AUX_STACK_OVERFLOW, + EXCE_AUX_STACK_UNDERFLOW, + EXCE_OUT_OF_BOUNDS_TABLE_ACCESS, + EXCE_OPERAND_STACK_OVERFLOW, + EXCE_ALREADY_THROWN, + EXCE_NUM, +} WASMExceptionID; + +typedef union { + uint64 u64; + uint32 u32[2]; +} MemBound; + struct WASMMemoryInstance { /* Module type */ uint32 module_type; @@ -35,59 +82,40 @@ struct WASMMemoryInstance { uint32 max_page_count; /* Memory data size */ uint32 memory_data_size; - /** * Memory data begin address, Note: * the app-heap might be inserted in to the linear memory, * when memory is re-allocated, the heap data and memory data * must be copied to new memory also */ - uint8 *memory_data; + DefPointer(uint8 *, memory_data); /* Memory data end address */ - uint8 *memory_data_end; + DefPointer(uint8 *, memory_data_end); /* Heap data base address */ - uint8 *heap_data; + DefPointer(uint8 *, heap_data); /* Heap data end address */ - uint8 *heap_data_end; + DefPointer(uint8 *, heap_data_end); /* The heap created */ - void *heap_handle; + DefPointer(void *, heap_handle); -#if WASM_ENABLE_SHARED_MEMORY != 0 - /* mutex lock for the memory, used in atomic operation */ - korp_mutex mem_lock; -#endif - -#if WASM_ENABLE_FAST_JIT != 0 -#if UINTPTR_MAX == UINT64_MAX - uint64 mem_bound_check_1byte; - uint64 mem_bound_check_2bytes; - uint64 mem_bound_check_4bytes; - uint64 mem_bound_check_8bytes; - uint64 mem_bound_check_16bytes; -#else - uint32 mem_bound_check_1byte; - uint32 mem_bound_check_2bytes; - uint32 mem_bound_check_4bytes; - uint32 mem_bound_check_8bytes; - uint32 mem_bound_check_16bytes; -#endif +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_AOT != 0 + MemBound mem_bound_check_1byte; + MemBound mem_bound_check_2bytes; + MemBound mem_bound_check_4bytes; + MemBound mem_bound_check_8bytes; + MemBound mem_bound_check_16bytes; #endif }; struct WASMTableInstance { - /* The element type, VALUE_TYPE_FUNCREF/EXTERNREF currently */ - uint8 elem_type; /* Current size */ uint32 cur_size; /* Maximum size */ uint32 max_size; -#if WASM_ENABLE_MULTI_MODULE != 0 - /* just for import, keep the reference here */ - WASMTableInstance *table_inst_linked; -#endif - /* Base address */ - uint8 base_addr[1]; + /* Table elements */ + uint32 elems[1]; }; struct WASMGlobalInstance { @@ -149,7 +177,6 @@ typedef struct WASMExportFuncInstance { WASMFunctionInstance *function; } WASMExportFuncInstance; -#if WASM_ENABLE_MULTI_MODULE != 0 typedef struct WASMExportGlobInstance { char *name; WASMGlobalInstance *global; @@ -164,8 +191,40 @@ typedef struct WASMExportMemInstance { char *name; WASMMemoryInstance *memory; } WASMExportMemInstance; + +/* Extra info of WASM module instance for interpreter/jit mode */ +typedef struct WASMModuleInstanceExtra { + WASMGlobalInstance *globals; + WASMFunctionInstance *functions; + + uint32 global_count; + uint32 function_count; + + WASMFunctionInstance *start_function; + WASMFunctionInstance *malloc_function; + WASMFunctionInstance *free_function; + WASMFunctionInstance *retain_function; + +#if WASM_ENABLE_SHARED_MEMORY != 0 + /* lock for shared memory atomic operations */ + korp_mutex mem_lock; + bool mem_lock_inited; #endif +#if WASM_ENABLE_MULTI_MODULE != 0 + bh_list sub_module_inst_list_head; + bh_list *sub_module_inst_list; + /* linked table instances of import table instances */ + WASMTableInstance **table_insts_linked; +#endif + +#if WASM_ENABLE_MEMORY_PROFILING != 0 + uint32 max_aux_stack_used; +#endif +} WASMModuleInstanceExtra; + +struct AOTFuncPerfProfInfo; + struct WASMModuleInstance { /* Module instance type, for module instance loaded from WASM bytecode binary, this field is Wasm_Module_Bytecode; @@ -175,78 +234,82 @@ struct WASMModuleInstance { uint32 module_type; uint32 memory_count; + DefPointer(WASMMemoryInstance **, memories); + + /* global and table info */ + uint32 global_data_size; uint32 table_count; - uint32 global_count; - uint32 function_count; + DefPointer(uint8 *, global_data); + /* For AOTModuleInstance, it denotes `AOTTableInstance *` */ + DefPointer(WASMTableInstance **, tables); + + /* import func ptrs + llvm jit func ptrs */ + DefPointer(void **, func_ptrs); + + /* function type indexes */ + DefPointer(uint32 *, func_type_indexes); uint32 export_func_count; -#if WASM_ENABLE_MULTI_MODULE != 0 - uint32 export_glob_count; - uint32 export_mem_count; - uint32 export_tab_count; -#endif - - /* Array of function pointers to import functions */ - void **import_func_ptrs; -#if WASM_ENABLE_FAST_JIT != 0 - /* point to JITed functions */ - void **fast_jit_func_ptrs; - uint32 *func_type_indexes; -#endif - - WASMMemoryInstance **memories; - WASMTableInstance **tables; - WASMGlobalInstance *globals; - WASMFunctionInstance *functions; - - WASMExportFuncInstance *export_functions; -#if WASM_ENABLE_MULTI_MODULE != 0 - WASMExportGlobInstance *export_globals; - WASMExportMemInstance *export_memories; - WASMExportTabInstance *export_tables; -#endif - - WASMMemoryInstance *default_memory; - WASMTableInstance *default_table; - /* Global data of global instances */ - uint8 *global_data; - - WASMFunctionInstance *start_function; - WASMFunctionInstance *malloc_function; - WASMFunctionInstance *free_function; - WASMFunctionInstance *retain_function; - - WASMModule *module; - -#if WASM_ENABLE_LIBC_WASI != 0 - WASIContext *wasi_ctx; -#endif - - WASMExecEnv *exec_env_singleton; - - /* Default WASM stack size of threads of this Module instance. */ - uint32 default_wasm_stack_size; + uint32 export_global_count; + uint32 export_memory_count; + uint32 export_table_count; + /* For AOTModuleInstance, it denotes `AOTFunctionInstance *` */ + DefPointer(WASMExportFuncInstance *, export_functions); + DefPointer(WASMExportGlobInstance *, export_globals); + DefPointer(WASMExportMemInstance *, export_memories); + DefPointer(WASMExportTabInstance *, export_tables); /* The exception buffer of wasm interpreter for current thread. */ char cur_exception[128]; -#if WASM_ENABLE_DUMP_CALL_STACK != 0 - Vector *frames; -#endif + /* The WASM module or AOT module, for AOTModuleInstance, + it denotes `AOTModule *` */ + DefPointer(WASMModule *, module); - /* The custom data that can be set/get by - * wasm_set_custom_data/wasm_get_custom_data */ - void *custom_data; - -#if WASM_ENABLE_MULTI_MODULE != 0 - /* TODO: add mutex for mutli-threads? */ - bh_list sub_module_inst_list_head; - bh_list *sub_module_inst_list; +#if WASM_ENABLE_LIBC_WASI + /* WASI context */ + DefPointer(WASIContext *, wasi_ctx); +#else + DefPointer(void *, wasi_ctx); #endif + DefPointer(WASMExecEnv *, exec_env_singleton); + /* Array of function pointers to import functions, + not available in AOTModuleInstance */ + DefPointer(void **, import_func_ptrs); + /* Array of function pointers to fast jit functions, + not available in AOTModuleInstance */ + DefPointer(void **, fast_jit_func_ptrs); + /* The custom data that can be set/get by wasm_{get|set}_custom_data */ + DefPointer(void *, custom_data); + /* Stack frames, used in call stack dump and perf profiling */ + DefPointer(Vector *, frames); + /* Function performance profiling info list, only available + in AOTModuleInstance */ + DefPointer(struct AOTFuncPerfProfInfo *, func_perf_profilings); + /* WASM/AOT module extra info, for AOTModuleInstance, + it denotes `AOTModuleInstanceExtra *` */ + DefPointer(WASMModuleInstanceExtra *, e); -#if WASM_ENABLE_MEMORY_PROFILING != 0 - uint32 max_aux_stack_used; -#endif + /* Default WASM operand stack size */ + uint32 default_wasm_stack_size; + uint32 reserved[3]; + + /* + * +------------------------------+ <-- memories + * | WASMMemoryInstance[mem_count], mem_count is always 1 for LLVM JIT/AOT + * +------------------------------+ <-- global_data + * | global data + * +------------------------------+ <-- tables + * | WASMTableInstance[table_count] + * +------------------------------+ <-- e + * | WASMModuleInstanceExtra + * +------------------------------+ + */ + union { + uint64 _make_it_8_byte_aligned_; + WASMMemoryInstance memory_instances[1]; + uint8 bytes[1]; + } global_table_data; }; struct WASMInterpFrame; @@ -342,12 +405,12 @@ wasm_create_exec_env_and_call_function(WASMModuleInstance *module_inst, WASMFunctionInstance *function, unsigned argc, uint32 argv[]); -bool -wasm_create_exec_env_singleton(WASMModuleInstance *module_inst); - void wasm_set_exception(WASMModuleInstance *module, const char *exception); +void +wasm_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id); + const char * wasm_get_exception(WASMModuleInstance *module); @@ -366,45 +429,25 @@ uint32 wasm_module_dup_data(WASMModuleInstance *module_inst, const char *src, uint32 size); +/** + * Check whether the app address and the buf is inside the linear memory, + * and convert the app address into native address + */ bool -wasm_validate_app_addr(WASMModuleInstance *module_inst, uint32 app_offset, - uint32 size); +wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, + uint32 app_buf_addr, uint32 app_buf_size, + void **p_native_addr); + +WASMMemoryInstance * +wasm_get_default_memory(WASMModuleInstance *module_inst); bool -wasm_validate_app_str_addr(WASMModuleInstance *module_inst, uint32 app_offset); - -bool -wasm_validate_native_addr(WASMModuleInstance *module_inst, void *native_ptr, - uint32 size); - -void * -wasm_addr_app_to_native(WASMModuleInstance *module_inst, uint32 app_offset); - -uint32 -wasm_addr_native_to_app(WASMModuleInstance *module_inst, void *native_ptr); - -bool -wasm_get_app_addr_range(WASMModuleInstance *module_inst, uint32 app_offset, - uint32 *p_app_start_offset, uint32 *p_app_end_offset); - -bool -wasm_get_native_addr_range(WASMModuleInstance *module_inst, uint8 *native_ptr, - uint8 **p_native_start_addr, - uint8 **p_native_end_addr); - -bool -wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count); +wasm_enlarge_memory(WASMModuleInstance *module_inst, uint32 inc_page_count); bool wasm_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, uint32 argc, uint32 argv[]); -#if WASM_ENABLE_FAST_JIT != 0 -bool -jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, - uint32 type_idx, uint32 argc, uint32 argv[]); -#endif - #if WASM_ENABLE_THREAD_MGR != 0 bool wasm_set_aux_stack(WASMExecEnv *exec_env, uint32 start_offset, uint32 size); @@ -413,16 +456,6 @@ bool wasm_get_aux_stack(WASMExecEnv *exec_env, uint32 *start_offset, uint32 *size); #endif -#ifdef OS_ENABLE_HW_BOUND_CHECK -#ifndef BH_PLATFORM_WINDOWS -void -wasm_signal_handler(WASMSignalInfo *sig_info); -#else -LONG -wasm_exception_handler(WASMSignalInfo *sig_info); -#endif -#endif - void wasm_get_module_mem_consumption(const WASMModule *module, WASMModuleMemConsumption *mem_conspn); @@ -456,13 +489,14 @@ wasm_enlarge_table(WASMModuleInstance *module_inst, uint32 table_idx, #endif /* WASM_ENABLE_REF_TYPES != 0 */ static inline WASMTableInstance * -wasm_get_table_inst(const WASMModuleInstance *module_inst, const uint32 tbl_idx) +wasm_get_table_inst(const WASMModuleInstance *module_inst, uint32 tbl_idx) { /* careful, it might be a table in another module */ WASMTableInstance *tbl_inst = module_inst->tables[tbl_idx]; #if WASM_ENABLE_MULTI_MODULE != 0 - if (tbl_inst->table_inst_linked) { - tbl_inst = tbl_inst->table_inst_linked; + if (tbl_idx < module_inst->module->import_table_count + && module_inst->e->table_insts_linked[tbl_idx]) { + tbl_inst = module_inst->e->table_insts_linked[tbl_idx]; } #endif bh_assert(tbl_inst); @@ -495,6 +529,82 @@ const uint8 * wasm_loader_get_custom_section(WASMModule *module, const char *name, uint32 *len); +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 +void +jit_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id); + +/** + * Check whether the app address and the buf is inside the linear memory, + * and convert the app address into native address + */ +bool +jit_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, + uint32 app_buf_addr, uint32 app_buf_size, + void **p_native_addr); +#endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 */ + +#if WASM_ENABLE_FAST_JIT != 0 +bool +fast_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 type_idx, uint32 argc, uint32 *argv); + +bool +fast_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, + struct WASMInterpFrame *prev_frame); +#endif + +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 +bool +llvm_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 argc, uint32 *argv); + +bool +llvm_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc, + uint32 *argv); + +#if WASM_ENABLE_BULK_MEMORY != 0 +bool +llvm_jit_memory_init(WASMModuleInstance *module_inst, uint32 seg_index, + uint32 offset, uint32 len, uint32 dst); + +bool +llvm_jit_data_drop(WASMModuleInstance *module_inst, uint32 seg_index); +#endif + +#if WASM_ENABLE_REF_TYPES != 0 +void +llvm_jit_drop_table_seg(WASMModuleInstance *module_inst, uint32 tbl_seg_idx); + +void +llvm_jit_table_init(WASMModuleInstance *module_inst, uint32 tbl_idx, + uint32 tbl_seg_idx, uint32 length, uint32 src_offset, + uint32 dst_offset); + +void +llvm_jit_table_copy(WASMModuleInstance *module_inst, uint32 src_tbl_idx, + uint32 dst_tbl_idx, uint32 length, uint32 src_offset, + uint32 dst_offset); + +void +llvm_jit_table_fill(WASMModuleInstance *module_inst, uint32 tbl_idx, + uint32 length, uint32 val, uint32 data_offset); + +uint32 +llvm_jit_table_grow(WASMModuleInstance *module_inst, uint32 tbl_idx, + uint32 inc_entries, uint32 init_val); +#endif + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 || WASM_ENABLE_PERF_PROFILING != 0 +bool +llvm_jit_alloc_frame(WASMExecEnv *exec_env, uint32 func_index); + +void +llvm_jit_free_frame(WASMExecEnv *exec_env); +#endif +#endif /* end of WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 */ + #ifdef __cplusplus } #endif diff --git a/core/iwasm/libraries/debug-engine/debug_engine.c b/core/iwasm/libraries/debug-engine/debug_engine.c index e62ba5a2..10e39b57 100644 --- a/core/iwasm/libraries/debug-engine/debug_engine.c +++ b/core/iwasm/libraries/debug-engine/debug_engine.c @@ -617,7 +617,7 @@ wasm_debug_instance_get_memregion(WASMDebugInstance *instance, uint64 addr) break; case WasmMemory: { - memory = module_inst->default_memory; + memory = wasm_get_default_memory(module_inst); if (memory) { num_bytes_per_page = memory->num_bytes_per_page; @@ -715,7 +715,7 @@ wasm_debug_instance_get_linear_mem(WASMDebugInstance *instance, uint64 offset, return false; module_inst = (WASMModuleInstance *)exec_env->module_inst; - memory = module_inst->default_memory; + memory = wasm_get_default_memory(module_inst); if (memory) { num_bytes_per_page = memory->num_bytes_per_page; linear_mem_size = num_bytes_per_page * memory->cur_page_count; @@ -748,7 +748,7 @@ wasm_debug_instance_set_linear_mem(WASMDebugInstance *instance, uint64 offset, return false; module_inst = (WASMModuleInstance *)exec_env->module_inst; - memory = module_inst->default_memory; + memory = wasm_get_default_memory(module_inst); if (memory) { num_bytes_per_page = memory->num_bytes_per_page; linear_mem_size = num_bytes_per_page * memory->cur_page_count; @@ -1134,10 +1134,10 @@ wasm_debug_instance_get_global(WASMDebugInstance *instance, int32 frame_index, module_inst = (WASMModuleInstance *)exec_env->module_inst; global_data = module_inst->global_data; - globals = module_inst->globals; + globals = module_inst->e->globals; if ((global_index < 0) - || ((uint32)global_index >= module_inst->global_count)) { + || ((uint32)global_index >= module_inst->e->global_count)) { return false; } global = globals + global_index; diff --git a/product-mini/platforms/android/CMakeLists.txt b/product-mini/platforms/android/CMakeLists.txt index a65b1808..638b6ab0 100644 --- a/product-mini/platforms/android/CMakeLists.txt +++ b/product-mini/platforms/android/CMakeLists.txt @@ -23,7 +23,8 @@ set (WAMR_BUILD_PLATFORM "android") set (WAMR_BUILD_TARGET "X86_32") set (WAMR_BUILD_TYPE Release) set (WAMR_BUILD_INTERP 1) -set (WAMR_BUILD_AOT 0) +set (WAMR_BUILD_AOT 1) +set (WAMR_BUILD_JIT 0) set (WAMR_BUILD_LIBC_BUILTIN 1) set (WAMR_BUILD_LIBC_WASI 1) diff --git a/samples/wasm-c-api/CMakeLists.txt b/samples/wasm-c-api/CMakeLists.txt index 63fb06e7..05b910a5 100644 --- a/samples/wasm-c-api/CMakeLists.txt +++ b/samples/wasm-c-api/CMakeLists.txt @@ -125,6 +125,7 @@ set(MM_UTIL src/utils/multi_module_utils.c) set(EXAMPLES callback callback_chain + clone empty_imports global hello @@ -135,6 +136,12 @@ set(EXAMPLES trap ) +if(WAMR_BUILD_JIT AND WAMR_BUILD_LAZY_JIT) + if((${WAMR_BUILD_JIT} EQUAL 1) AND (${WAMR_BUILD_LAZY_JIT} EQUAL 1)) + list(APPEND EXAMPLES serialize) + endif() +endif() + foreach(EX ${EXAMPLES}) set(SRC ${CMAKE_CURRENT_LIST_DIR}/src/${EX}.c) diff --git a/samples/wasm-c-api/src/clone.c b/samples/wasm-c-api/src/clone.c new file mode 100644 index 00000000..c27dc680 --- /dev/null +++ b/samples/wasm-c-api/src/clone.c @@ -0,0 +1,518 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include + +#include "wasm_c_api.h" + +#define WORKER_NUMBER 10 + +/******************************* VM *******************************/ +/* Use wasm_vm_t and vm_xxx to simulate a minimal Wasm VM in Envoy */ + +typedef struct _vm { + wasm_engine_t *engine; + wasm_store_t *store; + wasm_module_t *module; + wasm_instance_t *instance; + wasm_func_t **function_list; + wasm_memory_t *memory; + wasm_table_t *table; + wasm_extern_vec_t *exports; +} wasm_vm_t; + +typedef enum _clone_level { + not_cloneable = 0, + compiled_bytecode, + instantiated_module +} clone_level; + +typedef struct _thread_arg_t { + char name[32]; + bool *ready_go_flag; + pthread_mutex_t *ready_go_lock; + pthread_cond_t *ready_go_cond; + const wasm_vm_t *base_vm; +} thread_arg_t; + +wasm_vm_t * +vm_new() +{ + wasm_vm_t *vm = NULL; + + vm = malloc(sizeof(struct _vm)); + if (!vm) + goto fail; + + memset(vm, 0, sizeof(wasm_vm_t)); + + vm->engine = wasm_engine_new(); + if (!vm->engine) + goto fail; + + vm->store = wasm_store_new(vm->engine); + if (!vm->store) + goto fail; + + return vm; + +fail: + if (vm) { + if (vm->engine) + wasm_engine_delete(vm->engine); + + free(vm); + } + return NULL; +} + +bool +vm_load(wasm_vm_t *vm, const wasm_byte_vec_t *binary) +{ + vm->module = wasm_module_new(vm->store, binary); + return vm->module != NULL; +} + +bool +vm_link(wasm_vm_t *vm, wasm_extern_vec_t *imports) +{ + vm->instance = wasm_instance_new(vm->store, vm->module, imports, NULL); + if (!vm->instance) + goto fail; + + vm->exports = malloc(sizeof(wasm_extern_vec_t)); + if (!vm->exports) + goto fail; + + memset(vm->exports, 0, sizeof(wasm_extern_vec_t)); + wasm_instance_exports(vm->instance, vm->exports); + /* an exported memory, and two exported functions */ + assert(vm->exports->size == 3); + + /* bind memory */ + assert(wasm_extern_kind(vm->exports->data[0]) == WASM_EXTERN_MEMORY); + vm->memory = wasm_extern_as_memory(vm->exports->data[0]); + + vm->function_list = malloc(2 * sizeof(wasm_func_t *)); + if (!vm->function_list) + goto fail; + + memset(vm->function_list, 0, sizeof(2 * sizeof(wasm_func_t *))); + + /* bind wasm_set_byte(...) */ + assert(wasm_extern_kind(vm->exports->data[1]) == WASM_EXTERN_FUNC); + vm->function_list[0] = wasm_extern_as_func(vm->exports->data[1]); + + /* bind wasm_get_byte(...) */ + assert(wasm_extern_kind(vm->exports->data[2]) == WASM_EXTERN_FUNC); + vm->function_list[1] = wasm_extern_as_func(vm->exports->data[2]); + + return true; +fail: + return false; +} + +wasm_vm_t * +vm_clone_from_module(const wasm_vm_t *base) +{ + printf("Initializing...\n"); + wasm_vm_t *secondary = NULL; + + secondary = vm_new(); + if (secondary) { + printf("Reuse module and bypass vm_load()..."); + secondary->module = base->module; + } + + return secondary; +} + +wasm_vm_t * +vm_clone_from_instance(const wasm_vm_t *base) +{ + /** + * if do a clone of the level instantiated_module, need to malloc and + * initialie + * - global. WASMGlobalIntance and global data + * - memory. WAAMMemoryInstance, memory_data and heap + * - table. WASMTableInstance, table_data + * - exports. all global, memory and table + * + * it is almost everything in wasm_instantiate() except funciton. + */ + (void)base; + printf("Unsupported\n"); + return NULL; +} + +wasm_vm_t * +vm_clone(const wasm_vm_t *base, clone_level level) +{ + if (level == not_cloneable) + return NULL; + + if (level == compiled_bytecode) + return vm_clone_from_module(base); + else + return vm_clone_from_instance(base); +} + +wasm_vm_t * +vm_release(wasm_vm_t *vm) +{ + if (!vm) + return NULL; + + if (vm->function_list) { + free(vm->function_list); + vm->function_list = NULL; + } + + if (vm->exports) { + wasm_extern_vec_delete(vm->exports); + free(vm->exports); + vm->exports = NULL; + } + + wasm_instance_delete(vm->instance); + vm->instance = NULL; + + wasm_module_delete(vm->module); + vm->module = NULL; + + wasm_store_delete(vm->store); + vm->store = NULL; + + wasm_engine_delete(vm->engine); + vm->engine = NULL; + + free(vm); + return NULL; +} + +bool +vm_memory_set_byte(const wasm_vm_t *vm, uint32_t offset, uint8_t byte) +{ + byte_t *data = wasm_memory_data(vm->memory); + assert(data); + *(data + offset) = byte; + return true; +} + +bool +vm_memory_get_byte(const wasm_vm_t *vm, uint32_t offset, uint8_t *byte) +{ + byte_t *data = wasm_memory_data(vm->memory); + assert(data); + *byte = *(data + offset); + return true; +} + +bool +vm_function_set_byte(const wasm_vm_t *vm, uint32_t offset, uint8_t byte) +{ + wasm_val_t a_v[2] = { WASM_I32_VAL(offset), WASM_I32_VAL(byte) }; + wasm_val_vec_t args = WASM_ARRAY_VEC(a_v); + wasm_val_vec_t results = WASM_EMPTY_VEC; + wasm_trap_t *trap = wasm_func_call(vm->function_list[0], &args, &results); + if (trap) + printf("call wasm_set_byte failed"); + + return trap != NULL; +} + +bool +vm_function_get_byte(const wasm_vm_t *vm, uint32_t offset, uint8_t *byte) +{ + wasm_val_t a_v[1] = { WASM_I32_VAL(offset) }; + wasm_val_vec_t args = WASM_ARRAY_VEC(a_v); + wasm_val_t r_v[1] = { WASM_INIT_VAL }; + wasm_val_vec_t results = WASM_ARRAY_VEC(r_v); + wasm_trap_t *trap = wasm_func_call(vm->function_list[1], &args, &results); + if (trap) { + printf("call wasm_get_byte failed"); + return false; + } + + assert(results.data->kind == WASM_I32); + *byte = results.data->of.i32; + return true; +} + +static bool +load_wasm_file_content(const char *file_name, wasm_byte_vec_t *out) +{ + bool ret = false; +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_INTERP == 0 + FILE *file = fopen(file_name, "rb"); +#else + FILE *file = fopen(file_name, "rb"); +#endif + if (!file) { + printf("> Error loading .wasm!\n"); + goto quit; + } + + int offset = fseek(file, 0L, SEEK_END); + if (offset == -1) { + printf("> Error loading .wasm!\n"); + goto close_file; + } + + long file_size = ftell(file); + if (file_size == -1) { + printf("> Error loading .wasm!\n"); + goto close_file; + } + + offset = fseek(file, 0L, SEEK_SET); + if (offset == -1) { + printf("> Error loading .wasm!\n"); + goto close_file; + } + + wasm_byte_vec_new_uninitialized(out, file_size); + if (fread(out->data, file_size, 1, file) != 1) { + printf("> Error loading content!\n"); + goto close_file; + } + + ret = true; +close_file: + fclose(file); +quit: + return ret; +} + +static pthread_key_t name_key; + +wasm_trap_t * +report_cb(const wasm_val_vec_t *args, wasm_val_vec_t *results) +{ + (void)results; + + assert(args->data[0].kind == WASM_I32); + uint32_t chk_pnt_no = args->data[0].of.i32; + + char *name = pthread_getspecific(name_key); + printf("[%s] Pass CHK POINT #%u\n", name, chk_pnt_no); + + return NULL; +} + +bool +run_code_start(wasm_vm_t **out) +{ + bool ret = false; + + printf("Initializing...\n"); + wasm_vm_t *vm = vm_new(); + if (!vm) + goto fail; + + printf("Loading binary...\n"); + wasm_byte_vec_t binary = { 0 }; +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_INTERP == 0 + const char *file_name = "clone.aot"; +#else + const char *file_name = "clone.wasm"; +#endif + if (!load_wasm_file_content(file_name, &binary)) + goto release_vm; + + printf("Compiling module...\n"); + ret = vm_load(vm, &binary); + wasm_byte_vec_delete(&binary); + if (!ret) + goto release_vm; + + printf("Creating callback...\n"); + wasm_functype_t *callback_type = + wasm_functype_new_1_0(wasm_valtype_new_i32()); + if (!callback_type) + goto release_vm; + + wasm_func_t *callback = wasm_func_new(vm->store, callback_type, report_cb); + wasm_functype_delete(callback_type); + if (!callback) + goto release_vm; + + printf("Instantiating module...\n"); + wasm_extern_t *externs[] = { wasm_func_as_extern(callback) }; + wasm_extern_vec_t imports = WASM_ARRAY_VEC(externs); + ret = vm_link(vm, &imports); + wasm_func_delete(callback); + if (!ret) + goto release_vm; + + *out = vm; + return true; + +release_vm: + vm_release(vm); +fail: + return false; +} + +bool +run_warm_start_w_compiled_bytecode(const wasm_vm_t *first, wasm_vm_t **out) +{ + bool ret; + wasm_vm_t *secondary = vm_clone(first, compiled_bytecode); + if (!secondary) + goto fail; + + printf("Creating callback...\n"); + wasm_functype_t *callback_type = + wasm_functype_new_1_0(wasm_valtype_new_i32()); + if (!callback_type) + goto release_vm; + + wasm_func_t *callback = + wasm_func_new(secondary->store, callback_type, report_cb); + wasm_functype_delete(callback_type); + if (!callback) + goto release_vm; + + printf("Instantiating module...\n"); + wasm_extern_t *externs[] = { wasm_func_as_extern(callback) }; + wasm_extern_vec_t imports = WASM_ARRAY_VEC(externs); + ret = vm_link(secondary, &imports); + wasm_func_delete(callback); + if (!ret) + goto release_vm; + + *out = secondary; + return true; + +release_vm: + vm_release(secondary); +fail: + return false; +} + +bool +run_warm_start_w_instantiated_module(const wasm_vm_t *first, wasm_vm_t **out) +{ + wasm_vm_t *secondary = vm_clone(first, instantiated_module); + if (!secondary) + return false; + + *out = secondary; + return true; +} + +void +run_test(const wasm_vm_t *vm) +{ + uint8_t byte = 0xFF; + + /* read initialization */ + vm_function_get_byte(vm, 10, &byte); + assert(byte == 0x0); + vm_memory_get_byte(vm, 10, &byte); + assert(byte == 0x0); + + /* read after writing */ + vm_function_set_byte(vm, 16, 0xab); + vm_function_get_byte(vm, 16, &byte); + assert(byte == 0xab); + + vm_memory_set_byte(vm, 16, 0xcd); + vm_memory_get_byte(vm, 16, &byte); + assert(byte == 0xcd); + + /* reading and writing across */ + vm_function_set_byte(vm, 16, 0xef); + vm_memory_get_byte(vm, 16, &byte); + assert(byte == 0xef); + + vm_memory_set_byte(vm, 16, 0x67); + vm_function_get_byte(vm, 16, &byte); + assert(byte == 0x67); + + printf("All Passed ...\n"); +} + +static void * +thrd_func(void *arg) +{ + thread_arg_t *thrd_arg = (thread_arg_t *)arg; + printf("Running warm start at %s...\n", thrd_arg->name); + + pthread_setspecific(name_key, thrd_arg->name); + + wasm_vm_t *vm; + if (!run_warm_start_w_compiled_bytecode(thrd_arg->base_vm, &vm)) + return NULL; + + pthread_mutex_trylock(thrd_arg->ready_go_lock); + while (!(*thrd_arg->ready_go_flag)) { + pthread_cond_wait(thrd_arg->ready_go_cond, thrd_arg->ready_go_lock); + } + pthread_mutex_unlock(thrd_arg->ready_go_lock); + + printf("Running test at %s...\n", thrd_arg->name); + run_test(vm); + + vm_release(vm); + pthread_exit(NULL); + return NULL; +} + +int +main() +{ + int ret = EXIT_FAILURE; + bool ready_go_flag = false; + pthread_mutex_t ready_go_lock = PTHREAD_MUTEX_INITIALIZER; + pthread_cond_t ready_go_cond = PTHREAD_COND_INITIALIZER; + pthread_key_create(&name_key, NULL); + pthread_setspecific(name_key, "Execution Thread"); + + printf("Running cold start at the execution thread...\n"); + wasm_vm_t *base_vm; + if (!run_code_start(&base_vm)) + goto quit; + run_test(base_vm); + + printf("Running warm start at other threads...\n"); + pthread_mutex_trylock(&ready_go_lock); + + pthread_t tids[WORKER_NUMBER] = { 0 }; + thread_arg_t thrd_args[WORKER_NUMBER] = { 0 }; + for (size_t i = 0; i < sizeof(tids) / sizeof(tids[0]); i++) { + thread_arg_t *thrd_arg = thrd_args + i; + + snprintf(thrd_arg->name, 32, "Worker#%lu", i); + thrd_arg->ready_go_cond = &ready_go_cond; + thrd_arg->ready_go_lock = &ready_go_lock; + thrd_arg->ready_go_flag = &ready_go_flag; + thrd_arg->base_vm = base_vm; + + int ret = pthread_create(&tids[i], NULL, thrd_func, thrd_arg); + if (ret != 0) + break; + } + + sleep(5); + + ready_go_flag = true; + pthread_mutex_unlock(&ready_go_lock); + pthread_cond_broadcast(&ready_go_cond); + + for (size_t i = 0; i < sizeof(tids) / sizeof(tids[0]); i++) { + if (tids[i] != 0) + pthread_join(tids[i], NULL); + } + vm_release(base_vm); + ret = EXIT_SUCCESS; +quit: + return ret; +} diff --git a/samples/wasm-c-api/src/clone.wat b/samples/wasm-c-api/src/clone.wat new file mode 100644 index 00000000..e9934cc0 --- /dev/null +++ b/samples/wasm-c-api/src/clone.wat @@ -0,0 +1,15 @@ +(module + (func $report (import "" "report") (param i32)) + + (memory (export "mem") 1 1) + + (func $wasm_set_byte (export "set_byte") (param i32 i32) + (call $report (i32.const 1)) + (i32.store8 (local.get 0) (local.get 1)) + ) + + (func $wasm_get_byte (export "get_byte") (param i32) (result i32) + (call $report (i32.const 2)) + (i32.load(local.get 0)) + ) +) \ No newline at end of file diff --git a/samples/wasm-c-api/src/serialize.c b/samples/wasm-c-api/src/serialize.c new file mode 100644 index 00000000..d6570691 --- /dev/null +++ b/samples/wasm-c-api/src/serialize.c @@ -0,0 +1,129 @@ +#include +#include +#include +#include + +#include "wasm_c_api.h" + +#define own + +// A function to be called from Wasm code. +own wasm_trap_t * +hello_callback(const wasm_val_vec_t *args, wasm_val_vec_t *results) +{ + printf("Calling back...\n"); + printf("> Hello World!\n"); + return NULL; +} + +int +main(int argc, const char *argv[]) +{ + // Initialize. + printf("Initializing...\n"); + wasm_engine_t *engine = wasm_engine_new(); + wasm_store_t *store = wasm_store_new(engine); + + // Load binary. + printf("Loading binary...\n"); + FILE *file = fopen("serialize.wasm", "rb"); + if (!file) { + printf("> Error loading module!\n"); + return 1; + } + fseek(file, 0L, SEEK_END); + size_t file_size = ftell(file); + fseek(file, 0L, SEEK_SET); + wasm_byte_vec_t binary; + wasm_byte_vec_new_uninitialized(&binary, file_size); + if (fread(binary.data, file_size, 1, file) != 1) { + printf("> Error loading module!\n"); + return 1; + } + fclose(file); + + // Compile. + printf("Compiling module...\n"); + own wasm_module_t *module = wasm_module_new(store, &binary); + if (!module) { + printf("> Error compiling module!\n"); + return 1; + } + + wasm_byte_vec_delete(&binary); + + // Serialize module. + printf("Serializing module...\n"); + own wasm_byte_vec_t serialized; + wasm_module_serialize(module, &serialized); + + wasm_module_delete(module); + + // Deserialize module. + printf("Deserializing module...\n"); + own wasm_module_t *deserialized = + wasm_module_deserialize(store, &serialized); + if (!deserialized) { + printf("> Error deserializing module!\n"); + return 1; + } + + wasm_byte_vec_delete(&serialized); + + // Create external print functions. + printf("Creating callback...\n"); + own wasm_functype_t *hello_type = wasm_functype_new_0_0(); + own wasm_func_t *hello_func = + wasm_func_new(store, hello_type, hello_callback); + + wasm_functype_delete(hello_type); + + // Instantiate. + printf("Instantiating deserialized module...\n"); + wasm_extern_t *externs[] = { wasm_func_as_extern(hello_func) }; + wasm_extern_vec_t imports = WASM_ARRAY_VEC(externs); + own wasm_instance_t *instance = + wasm_instance_new(store, deserialized, &imports, NULL); + if (!instance) { + printf("> Error instantiating module!\n"); + return 1; + } + + wasm_func_delete(hello_func); + + // Extract export. + printf("Extracting export...\n"); + own wasm_extern_vec_t exports; + wasm_instance_exports(instance, &exports); + if (exports.size == 0) { + printf("> Error accessing exports!\n"); + return 1; + } + const wasm_func_t *run_func = wasm_extern_as_func(exports.data[0]); + if (run_func == NULL) { + printf("> Error accessing export!\n"); + return 1; + } + + wasm_module_delete(deserialized); + wasm_instance_delete(instance); + + // Call. + printf("Calling export...\n"); + wasm_val_vec_t empty = WASM_EMPTY_VEC; + if (wasm_func_call(run_func, &empty, &empty)) { + printf("> Error calling function!\n"); + return 1; + } + + wasm_extern_vec_delete(&exports); + + // Shut down. + printf("Shutting down...\n"); + wasm_store_delete(store); + wasm_engine_delete(engine); + + // All done. + printf("Done.\n"); + return 0; +} diff --git a/samples/wasm-c-api/src/serialize.wat b/samples/wasm-c-api/src/serialize.wat new file mode 100644 index 00000000..1c56c558 --- /dev/null +++ b/samples/wasm-c-api/src/serialize.wat @@ -0,0 +1,4 @@ +(module + (func $hello (import "" "hello")) + (func (export "run") (call $hello)) +)