724858c731
Update wasm app build scripts for wasi-sdk-12.0: add --export=__main_argc_argv, remove --no-threads Lookup function with name "__main_argc_argv" as main function besides "main" Change module_malloc to runtime_malloc in wasi native lib Refine classic interpreter op_block and op_br_table Refine faster interpreter op_br_table Signed-off-by: Wenyong Huang <wenyong.huang@intel.com>
1089 lines
40 KiB
C
1089 lines
40 KiB
C
/*
|
|
* Copyright (C) 2019 Intel Corporation. All rights reserved.
|
|
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
*/
|
|
|
|
#include "aot_emit_control.h"
|
|
#include "aot_emit_exception.h"
|
|
#include "../aot/aot_runtime.h"
|
|
#include "../interpreter/wasm_loader.h"
|
|
|
|
static char *block_name_prefix[] = { "block", "loop", "if" };
|
|
static char *block_name_suffix[] = { "begin", "else", "end" };
|
|
|
|
enum {
|
|
LABEL_BEGIN = 0,
|
|
LABEL_ELSE,
|
|
LABEL_END
|
|
};
|
|
|
|
static void
|
|
format_block_name(char *name, uint32 name_size,
|
|
uint32 block_index, uint32 label_type,
|
|
uint32 label_id)
|
|
{
|
|
if (label_type != LABEL_TYPE_FUNCTION)
|
|
snprintf(name, name_size, "%s%d%s%s",
|
|
block_name_prefix[label_type], block_index,
|
|
"_", block_name_suffix[label_id]);
|
|
else
|
|
snprintf(name, name_size, "%s", "func_end");
|
|
}
|
|
|
|
#define CREATE_BLOCK(new_llvm_block, name) do { \
|
|
if (!(new_llvm_block = \
|
|
LLVMAppendBasicBlockInContext(comp_ctx->context, \
|
|
func_ctx->func, \
|
|
name))) { \
|
|
aot_set_last_error("add LLVM basic block failed.");\
|
|
goto fail; \
|
|
} \
|
|
} while (0)
|
|
|
|
#define CURR_BLOCK() LLVMGetInsertBlock(comp_ctx->builder)
|
|
|
|
#define MOVE_BLOCK_AFTER(llvm_block, llvm_block_after) \
|
|
LLVMMoveBasicBlockAfter(llvm_block, llvm_block_after)
|
|
|
|
#define MOVE_BLOCK_AFTER_CURR(llvm_block) \
|
|
LLVMMoveBasicBlockAfter(llvm_block, CURR_BLOCK())
|
|
|
|
#define MOVE_BLOCK_BEFORE(llvm_block, llvm_block_before) \
|
|
LLVMMoveBasicBlockBefore(llvm_block, llvm_block_before)
|
|
|
|
#define BUILD_BR(llvm_block) do { \
|
|
if (!LLVMBuildBr(comp_ctx->builder, llvm_block)) { \
|
|
aot_set_last_error("llvm build br failed."); \
|
|
goto fail; \
|
|
} \
|
|
} while (0)
|
|
|
|
#define BUILD_COND_BR(value_if, block_then, block_else) do {\
|
|
if (!LLVMBuildCondBr(comp_ctx->builder, value_if, \
|
|
block_then, block_else)) { \
|
|
aot_set_last_error("llvm build cond br failed."); \
|
|
goto fail; \
|
|
} \
|
|
} while (0)
|
|
|
|
#define SET_BUILDER_POS(llvm_block) \
|
|
LLVMPositionBuilderAtEnd(comp_ctx->builder, llvm_block)
|
|
|
|
#define CREATE_RESULT_VALUE_PHIS(block) do { \
|
|
if (block->result_count && !block->result_phis) { \
|
|
uint32 i; \
|
|
uint64 size; \
|
|
LLVMBasicBlockRef block_curr = CURR_BLOCK(); \
|
|
/* Allocate memory */ \
|
|
size = sizeof(LLVMValueRef) * (uint64)block->result_count; \
|
|
if (size >= UINT32_MAX \
|
|
|| !(block->result_phis = \
|
|
wasm_runtime_malloc((uint32)size))) { \
|
|
aot_set_last_error("allocate memory failed."); \
|
|
goto fail; \
|
|
} \
|
|
SET_BUILDER_POS(block->llvm_end_block); \
|
|
for (i = 0; i < block->result_count; i++) { \
|
|
if (!(block->result_phis[i] = \
|
|
LLVMBuildPhi(comp_ctx->builder, \
|
|
TO_LLVM_TYPE(block->result_types[i]), \
|
|
"phi"))) { \
|
|
aot_set_last_error("llvm build phi failed."); \
|
|
goto fail; \
|
|
} \
|
|
} \
|
|
SET_BUILDER_POS(block_curr); \
|
|
} \
|
|
} while (0)
|
|
|
|
#define ADD_TO_RESULT_PHIS(block, value, idx) do { \
|
|
LLVMBasicBlockRef block_curr = CURR_BLOCK(); \
|
|
LLVMTypeRef phi_ty = LLVMTypeOf(block->result_phis[idx]); \
|
|
LLVMTypeRef value_ty = LLVMTypeOf(value); \
|
|
bh_assert(LLVMGetTypeKind(phi_ty) == LLVMGetTypeKind(value_ty)); \
|
|
bh_assert(LLVMGetTypeContext(phi_ty) \
|
|
== LLVMGetTypeContext(value_ty)); \
|
|
LLVMAddIncoming(block->result_phis[idx], &value, &block_curr, 1); \
|
|
(void)phi_ty; \
|
|
(void)value_ty; \
|
|
} while (0)
|
|
|
|
#define BUILD_ICMP(op, left, right, res, name) do { \
|
|
if (!(res = LLVMBuildICmp(comp_ctx->builder, op, \
|
|
left, right, name))) { \
|
|
aot_set_last_error("llvm build icmp failed."); \
|
|
goto fail; \
|
|
} \
|
|
} while (0)
|
|
|
|
#define ADD_TO_PARAM_PHIS(block, value, idx) do { \
|
|
LLVMBasicBlockRef block_curr = CURR_BLOCK(); \
|
|
LLVMAddIncoming(block->param_phis[idx], \
|
|
&value, &block_curr, 1); \
|
|
} while (0)
|
|
|
|
static LLVMBasicBlockRef
|
|
find_next_llvm_end_block(AOTBlock *block)
|
|
{
|
|
block = block->prev;
|
|
while (block && !block->llvm_end_block)
|
|
block = block->prev;
|
|
return block ? block->llvm_end_block : NULL;
|
|
}
|
|
|
|
static AOTBlock*
|
|
get_target_block(AOTFuncContext *func_ctx, uint32 br_depth)
|
|
{
|
|
uint32 i = br_depth;
|
|
AOTBlock *block = func_ctx->block_stack.block_list_end;
|
|
|
|
while (i-- > 0 && block) {
|
|
block = block->prev;
|
|
}
|
|
|
|
if (!block) {
|
|
aot_set_last_error("WASM block stack underflow.");
|
|
return NULL;
|
|
}
|
|
return block;
|
|
}
|
|
|
|
static bool
|
|
handle_next_reachable_block(AOTCompContext *comp_ctx,
|
|
AOTFuncContext *func_ctx,
|
|
uint8 **p_frame_ip)
|
|
{
|
|
AOTBlock *block = func_ctx->block_stack.block_list_end;
|
|
AOTBlock *block_prev;
|
|
uint8 *frame_ip = NULL;
|
|
uint32 i;
|
|
AOTFuncType *func_type;
|
|
|
|
aot_checked_addr_list_destroy(func_ctx);
|
|
|
|
bh_assert(block);
|
|
|
|
if (block->label_type == LABEL_TYPE_IF
|
|
&& block->llvm_else_block
|
|
&& !block->skip_wasm_code_else
|
|
&& *p_frame_ip <= block->wasm_code_else) {
|
|
/* Clear value stack and start to translate else branch */
|
|
aot_value_stack_destroy(&block->value_stack);
|
|
/* Recover parameters of else branch */
|
|
for (i = 0; i < block->param_count; i++)
|
|
PUSH(block->else_param_phis[i], block->param_types[i]);
|
|
SET_BUILDER_POS(block->llvm_else_block);
|
|
*p_frame_ip = block->wasm_code_else + 1;
|
|
return true;
|
|
}
|
|
|
|
while (block && !block->is_reachable) {
|
|
block_prev = block->prev;
|
|
block = aot_block_stack_pop(&func_ctx->block_stack);
|
|
|
|
if (block->label_type == LABEL_TYPE_IF) {
|
|
if (block->llvm_else_block
|
|
&& !block->skip_wasm_code_else
|
|
&& *p_frame_ip <= block->wasm_code_else) {
|
|
/* Clear value stack and start to translate else branch */
|
|
aot_value_stack_destroy(&block->value_stack);
|
|
SET_BUILDER_POS(block->llvm_else_block);
|
|
*p_frame_ip = block->wasm_code_else + 1;
|
|
/* Push back the block */
|
|
aot_block_stack_push(&func_ctx->block_stack, block);
|
|
return true;
|
|
}
|
|
else if (block->llvm_end_block) {
|
|
/* Remove unreachable basic block */
|
|
LLVMDeleteBasicBlock(block->llvm_end_block);
|
|
block->llvm_end_block = NULL;
|
|
}
|
|
}
|
|
|
|
frame_ip = block->wasm_code_end;
|
|
aot_block_destroy(block);
|
|
block = block_prev;
|
|
}
|
|
|
|
if (!block) {
|
|
*p_frame_ip = frame_ip + 1;
|
|
return true;
|
|
}
|
|
|
|
*p_frame_ip = block->wasm_code_end + 1;
|
|
SET_BUILDER_POS(block->llvm_end_block);
|
|
|
|
/* Pop block, push its return value, and destroy the block */
|
|
block = aot_block_stack_pop(&func_ctx->block_stack);
|
|
func_type = func_ctx->aot_func->func_type;
|
|
for (i = 0; i < block->result_count; i++) {
|
|
bh_assert(block->result_phis[i]);
|
|
if (block->label_type != LABEL_TYPE_FUNCTION) {
|
|
PUSH(block->result_phis[i], block->result_types[i]);
|
|
}
|
|
else {
|
|
/* Store extra return values to function parameters */
|
|
if (i != 0) {
|
|
uint32 param_index = func_type->param_count + i;
|
|
if (!LLVMBuildStore(comp_ctx->builder,
|
|
block->result_phis[i],
|
|
LLVMGetParam(func_ctx->func, param_index))) {
|
|
aot_set_last_error("llvm build store failed.");
|
|
goto fail;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if (block->label_type == LABEL_TYPE_FUNCTION) {
|
|
if (block->result_count) {
|
|
/* Return the first return value */
|
|
if (!LLVMBuildRet(comp_ctx->builder, block->result_phis[0])) {
|
|
aot_set_last_error("llvm build return failed.");
|
|
goto fail;
|
|
}
|
|
}
|
|
else {
|
|
if (!LLVMBuildRetVoid(comp_ctx->builder)) {
|
|
aot_set_last_error("llvm build return void failed.");
|
|
goto fail;
|
|
}
|
|
}
|
|
}
|
|
aot_block_destroy(block);
|
|
return true;
|
|
fail:
|
|
return false;
|
|
}
|
|
|
|
static bool
|
|
push_aot_block_to_stack_and_pass_params(AOTCompContext *comp_ctx,
|
|
AOTFuncContext *func_ctx,
|
|
AOTBlock *block)
|
|
{
|
|
uint32 i, param_index;
|
|
LLVMValueRef value;
|
|
uint64 size;
|
|
char name[32];
|
|
LLVMBasicBlockRef block_curr = CURR_BLOCK();
|
|
|
|
if (block->param_count) {
|
|
size = sizeof(LLVMValueRef) * (uint64)block->param_count;
|
|
if (size >= UINT32_MAX
|
|
|| !(block->param_phis = wasm_runtime_malloc((uint32)size))) {
|
|
aot_set_last_error("allocate memory failed.");
|
|
return false;
|
|
}
|
|
|
|
if (block->label_type == LABEL_TYPE_IF
|
|
&& !block->skip_wasm_code_else
|
|
&& !(block->else_param_phis = wasm_runtime_malloc((uint32)size))) {
|
|
wasm_runtime_free(block->param_phis);
|
|
block->param_phis = NULL;
|
|
aot_set_last_error("allocate memory failed.");
|
|
return false;
|
|
}
|
|
|
|
/* Create param phis */
|
|
for (i = 0; i < block->param_count; i++) {
|
|
SET_BUILDER_POS(block->llvm_entry_block);
|
|
snprintf(name, sizeof(name), "%s%d_phi%d",
|
|
block_name_prefix[block->label_type],
|
|
block->block_index, i);
|
|
if (!(block->param_phis[i] =
|
|
LLVMBuildPhi(comp_ctx->builder,
|
|
TO_LLVM_TYPE(block->param_types[i]),
|
|
name))) {
|
|
aot_set_last_error("llvm build phi failed.");
|
|
goto fail;
|
|
}
|
|
|
|
if (block->label_type == LABEL_TYPE_IF
|
|
&& !block->skip_wasm_code_else
|
|
&& block->llvm_else_block) {
|
|
/* Build else param phis */
|
|
SET_BUILDER_POS(block->llvm_else_block);
|
|
snprintf(name, sizeof(name), "else%d_phi%d",
|
|
block->block_index, i);
|
|
if (!(block->else_param_phis[i] =
|
|
LLVMBuildPhi(comp_ctx->builder,
|
|
TO_LLVM_TYPE(block->param_types[i]),
|
|
name))) {
|
|
aot_set_last_error("llvm build phi failed.");
|
|
goto fail;
|
|
}
|
|
}
|
|
}
|
|
SET_BUILDER_POS(block_curr);
|
|
|
|
/* Pop param values from current block's
|
|
* value stack and add to param phis.
|
|
*/
|
|
for (i = 0; i < block->param_count; i++) {
|
|
param_index = block->param_count - 1 - i;
|
|
POP(value, block->param_types[param_index]);
|
|
ADD_TO_PARAM_PHIS(block, value, param_index);
|
|
if (block->label_type == LABEL_TYPE_IF
|
|
&& !block->skip_wasm_code_else) {
|
|
if (block->llvm_else_block) {
|
|
/* has else branch, add to else param phis */
|
|
LLVMAddIncoming(block->else_param_phis[param_index],
|
|
&value, &block_curr, 1);
|
|
}
|
|
else {
|
|
/* no else branch, add to result phis */
|
|
CREATE_RESULT_VALUE_PHIS(block);
|
|
ADD_TO_RESULT_PHIS(block, value, param_index);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/* Push the new block to block stack */
|
|
aot_block_stack_push(&func_ctx->block_stack, block);
|
|
|
|
/* Push param phis to the new block */
|
|
for (i = 0; i < block->param_count; i++) {
|
|
PUSH(block->param_phis[i], block->param_types[i]);
|
|
}
|
|
|
|
return true;
|
|
|
|
fail:
|
|
if (block->param_phis) {
|
|
wasm_runtime_free(block->param_phis);
|
|
block->param_phis = NULL;
|
|
}
|
|
if (block->else_param_phis) {
|
|
wasm_runtime_free(block->else_param_phis);
|
|
block->else_param_phis = NULL;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
bool
|
|
aot_compile_op_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
|
uint8 **p_frame_ip, uint8 *frame_ip_end,
|
|
uint32 label_type, uint32 param_count, uint8 *param_types,
|
|
uint32 result_count, uint8 *result_types)
|
|
{
|
|
BlockAddr block_addr_cache[BLOCK_ADDR_CACHE_SIZE][BLOCK_ADDR_CONFLICT_SIZE];
|
|
AOTBlock *block;
|
|
uint8 *else_addr, *end_addr;
|
|
LLVMValueRef value;
|
|
char name[32];
|
|
|
|
/* Check block stack */
|
|
if (!func_ctx->block_stack.block_list_end) {
|
|
aot_set_last_error("WASM block stack underflow.");
|
|
return false;
|
|
}
|
|
|
|
memset(block_addr_cache, 0, sizeof(block_addr_cache));
|
|
|
|
/* Get block info */
|
|
if (!(wasm_loader_find_block_addr((BlockAddr*)block_addr_cache,
|
|
*p_frame_ip, frame_ip_end, (uint8)label_type,
|
|
&else_addr, &end_addr))) {
|
|
aot_set_last_error("find block end addr failed.");
|
|
return false;
|
|
}
|
|
|
|
/* Allocate memory */
|
|
if (!(block = wasm_runtime_malloc(sizeof(AOTBlock)))) {
|
|
aot_set_last_error("allocate memory failed.");
|
|
return false;
|
|
}
|
|
memset(block, 0, sizeof(AOTBlock));
|
|
if (param_count
|
|
&& !(block->param_types = wasm_runtime_malloc(param_count))) {
|
|
aot_set_last_error("allocate memory failed.");
|
|
goto fail;
|
|
}
|
|
if (result_count) {
|
|
if (!(block->result_types = wasm_runtime_malloc(result_count))) {
|
|
aot_set_last_error("allocate memory failed.");
|
|
goto fail;
|
|
}
|
|
}
|
|
|
|
/* Init aot block data */
|
|
block->label_type = label_type;
|
|
block->param_count = param_count;
|
|
memcpy(block->param_types, param_types, param_count);
|
|
block->result_count = result_count;
|
|
memcpy(block->result_types, result_types, result_count);
|
|
block->wasm_code_else = else_addr;
|
|
block->wasm_code_end = end_addr;
|
|
block->block_index = func_ctx->block_stack.block_index[label_type];
|
|
func_ctx->block_stack.block_index[label_type]++;
|
|
|
|
if (label_type == LABEL_TYPE_BLOCK
|
|
|| label_type == LABEL_TYPE_LOOP) {
|
|
/* Create block */
|
|
format_block_name(name, sizeof(name),
|
|
block->block_index, label_type, LABEL_BEGIN);
|
|
CREATE_BLOCK(block->llvm_entry_block, name);
|
|
MOVE_BLOCK_AFTER_CURR(block->llvm_entry_block);
|
|
/* Jump to the entry block */
|
|
BUILD_BR(block->llvm_entry_block);
|
|
if (!push_aot_block_to_stack_and_pass_params(comp_ctx, func_ctx, block))
|
|
goto fail;
|
|
/* Start to translate the block */
|
|
SET_BUILDER_POS(block->llvm_entry_block);
|
|
if (label_type == LABEL_TYPE_LOOP)
|
|
aot_checked_addr_list_destroy(func_ctx);
|
|
}
|
|
else if (label_type == LABEL_TYPE_IF) {
|
|
POP_COND(value);
|
|
if (!LLVMIsConstant(value)) {
|
|
/* Compare value is not constant, create condition br IR */
|
|
/* Create entry block */
|
|
format_block_name(name, sizeof(name),
|
|
block->block_index, label_type, LABEL_BEGIN);
|
|
CREATE_BLOCK(block->llvm_entry_block, name);
|
|
MOVE_BLOCK_AFTER_CURR(block->llvm_entry_block);
|
|
|
|
/* Create end block */
|
|
format_block_name(name, sizeof(name),
|
|
block->block_index, label_type, LABEL_END);
|
|
CREATE_BLOCK(block->llvm_end_block, name);
|
|
MOVE_BLOCK_AFTER(block->llvm_end_block, block->llvm_entry_block);
|
|
|
|
if (else_addr) {
|
|
/* Create else block */
|
|
format_block_name(name, sizeof(name),
|
|
block->block_index, label_type, LABEL_ELSE);
|
|
CREATE_BLOCK(block->llvm_else_block, name);
|
|
MOVE_BLOCK_AFTER(block->llvm_else_block, block->llvm_entry_block);
|
|
/* Create condition br IR */
|
|
BUILD_COND_BR(value, block->llvm_entry_block,
|
|
block->llvm_else_block);
|
|
}
|
|
else {
|
|
/* Create condition br IR */
|
|
BUILD_COND_BR(value, block->llvm_entry_block,
|
|
block->llvm_end_block);
|
|
block->is_reachable = true;
|
|
}
|
|
if (!push_aot_block_to_stack_and_pass_params(comp_ctx, func_ctx, block))
|
|
goto fail;
|
|
/* Start to translate if branch of BLOCK if */
|
|
SET_BUILDER_POS(block->llvm_entry_block);
|
|
}
|
|
else {
|
|
if ((int32)LLVMConstIntGetZExtValue(value) != 0) {
|
|
/* Compare value is not 0, condition is true, else branch of
|
|
BLOCK if cannot be reached */
|
|
block->skip_wasm_code_else = true;
|
|
/* Create entry block */
|
|
format_block_name(name, sizeof(name),
|
|
block->block_index, label_type, LABEL_BEGIN);
|
|
CREATE_BLOCK(block->llvm_entry_block, name);
|
|
MOVE_BLOCK_AFTER_CURR(block->llvm_entry_block);
|
|
/* Jump to the entry block */
|
|
BUILD_BR(block->llvm_entry_block);
|
|
if (!push_aot_block_to_stack_and_pass_params(comp_ctx, func_ctx, block))
|
|
goto fail;
|
|
/* Start to translate the if branch */
|
|
SET_BUILDER_POS(block->llvm_entry_block);
|
|
}
|
|
else {
|
|
/* Compare value is not 0, condition is false, if branch of
|
|
BLOCK if cannot be reached */
|
|
if (else_addr) {
|
|
/* Create else block */
|
|
format_block_name(name, sizeof(name),
|
|
block->block_index, label_type, LABEL_ELSE);
|
|
CREATE_BLOCK(block->llvm_else_block, name);
|
|
MOVE_BLOCK_AFTER_CURR(block->llvm_else_block);
|
|
/* Jump to the else block */
|
|
BUILD_BR(block->llvm_else_block);
|
|
if (!push_aot_block_to_stack_and_pass_params(comp_ctx, func_ctx, block))
|
|
goto fail;
|
|
/* Start to translate the else branch */
|
|
SET_BUILDER_POS(block->llvm_else_block);
|
|
*p_frame_ip = else_addr + 1;
|
|
}
|
|
else {
|
|
/* skip the block */
|
|
aot_block_destroy(block);
|
|
*p_frame_ip = end_addr + 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else {
|
|
aot_set_last_error("Invalid block type.");
|
|
goto fail;
|
|
}
|
|
|
|
return true;
|
|
fail:
|
|
aot_block_destroy(block);
|
|
return false;
|
|
}
|
|
|
|
bool
|
|
aot_compile_op_else(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
|
uint8 **p_frame_ip)
|
|
{
|
|
AOTBlock *block = func_ctx->block_stack.block_list_end;
|
|
LLVMValueRef value;
|
|
char name[32];
|
|
uint32 i, result_index;
|
|
|
|
/* Check block */
|
|
if (!block) {
|
|
aot_set_last_error("WASM block stack underflow.");
|
|
return false;
|
|
}
|
|
if (block->label_type != LABEL_TYPE_IF
|
|
|| (!block->skip_wasm_code_else
|
|
&& !block->llvm_else_block)) {
|
|
aot_set_last_error("Invalid WASM block type.");
|
|
return false;
|
|
}
|
|
|
|
/* Create end block if needed */
|
|
if (!block->llvm_end_block) {
|
|
format_block_name(name, sizeof(name),
|
|
block->block_index, block->label_type, LABEL_END);
|
|
CREATE_BLOCK(block->llvm_end_block, name);
|
|
if (block->llvm_else_block)
|
|
MOVE_BLOCK_AFTER(block->llvm_end_block, block->llvm_else_block);
|
|
else
|
|
MOVE_BLOCK_AFTER_CURR(block->llvm_end_block);
|
|
}
|
|
|
|
block->is_reachable = true;
|
|
|
|
/* Comes from the if branch of BLOCK if */
|
|
CREATE_RESULT_VALUE_PHIS(block);
|
|
for (i = 0; i < block->result_count; i++) {
|
|
result_index = block->result_count - 1 - i;
|
|
POP(value, block->result_types[result_index]);
|
|
ADD_TO_RESULT_PHIS(block, value, result_index);
|
|
}
|
|
|
|
/* Jump to end block */
|
|
BUILD_BR(block->llvm_end_block);
|
|
|
|
if (!block->skip_wasm_code_else
|
|
&& block->llvm_else_block) {
|
|
/* Clear value stack, recover param values
|
|
* and start to translate else branch.
|
|
*/
|
|
aot_value_stack_destroy(&block->value_stack);
|
|
for (i = 0; i < block->param_count; i++)
|
|
PUSH(block->else_param_phis[i], block->param_types[i]);
|
|
SET_BUILDER_POS(block->llvm_else_block);
|
|
aot_checked_addr_list_destroy(func_ctx);
|
|
return true;
|
|
}
|
|
|
|
/* No else branch or no need to translate else branch */
|
|
block->is_reachable = true;
|
|
return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip);
|
|
fail:
|
|
return false;
|
|
}
|
|
|
|
bool
|
|
aot_compile_op_end(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
|
uint8 **p_frame_ip)
|
|
{
|
|
AOTBlock *block;
|
|
LLVMValueRef value;
|
|
LLVMBasicBlockRef next_llvm_end_block;
|
|
char name[32];
|
|
uint32 i, result_index;
|
|
|
|
/* Check block stack */
|
|
if (!(block = func_ctx->block_stack.block_list_end)) {
|
|
aot_set_last_error("WASM block stack underflow.");
|
|
return false;
|
|
}
|
|
|
|
/* Create the end block */
|
|
if (!block->llvm_end_block) {
|
|
format_block_name(name, sizeof(name),
|
|
block->block_index, block->label_type, LABEL_END);
|
|
CREATE_BLOCK(block->llvm_end_block, name);
|
|
if ((next_llvm_end_block = find_next_llvm_end_block(block)))
|
|
MOVE_BLOCK_BEFORE(block->llvm_end_block, next_llvm_end_block);
|
|
}
|
|
|
|
/* Handle block result values */
|
|
CREATE_RESULT_VALUE_PHIS(block);
|
|
for (i = 0; i < block->result_count; i++) {
|
|
result_index = block->result_count - 1 - i;
|
|
POP(value, block->result_types[result_index]);
|
|
ADD_TO_RESULT_PHIS(block, value, result_index);
|
|
}
|
|
|
|
/* Jump to the end block */
|
|
BUILD_BR(block->llvm_end_block);
|
|
|
|
block->is_reachable = true;
|
|
return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip);
|
|
fail:
|
|
return false;
|
|
}
|
|
|
|
#if WASM_ENABLE_THREAD_MGR != 0
|
|
bool
|
|
check_suspend_flags(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
|
|
{
|
|
LLVMValueRef terminate_addr, terminate_flags, flag, offset, res;
|
|
LLVMBasicBlockRef terminate_check_block, non_terminate_block;
|
|
AOTFuncType *aot_func_type = func_ctx->aot_func->func_type;
|
|
LLVMBasicBlockRef terminate_block;
|
|
|
|
/* Offset of suspend_flags */
|
|
offset = I32_CONST(5);
|
|
CHECK_LLVM_CONST(offset);
|
|
|
|
if (!(terminate_addr =
|
|
LLVMBuildInBoundsGEP(comp_ctx->builder, func_ctx->exec_env,
|
|
&offset, 1, "terminate_addr"))) {
|
|
aot_set_last_error("llvm build in bounds gep failed");
|
|
return false;
|
|
}
|
|
if (!(terminate_addr =
|
|
LLVMBuildBitCast(comp_ctx->builder,
|
|
terminate_addr,
|
|
INT32_PTR_TYPE, "terminate_addr_ptr"))) {
|
|
aot_set_last_error("llvm build bit cast failed");
|
|
return false;
|
|
}
|
|
|
|
if (!(terminate_flags =
|
|
LLVMBuildLoad(comp_ctx->builder,
|
|
terminate_addr, "terminate_flags"))) {
|
|
aot_set_last_error("llvm build bit cast failed");
|
|
return false;
|
|
}
|
|
/* Set terminate_flags memory accecc to volatile, so that the value
|
|
will always be loaded from memory rather than register */
|
|
LLVMSetVolatile(terminate_flags, true);
|
|
|
|
CREATE_BLOCK(terminate_check_block, "terminate_check");
|
|
MOVE_BLOCK_AFTER_CURR(terminate_check_block);
|
|
|
|
CREATE_BLOCK(non_terminate_block, "non_terminate");
|
|
MOVE_BLOCK_AFTER_CURR(non_terminate_block);
|
|
|
|
BUILD_ICMP(LLVMIntSGT, terminate_flags, I32_ZERO, res, "need_terminate");
|
|
BUILD_COND_BR(res, terminate_check_block, non_terminate_block);
|
|
|
|
/* Move builder to terminate check block */
|
|
SET_BUILDER_POS(terminate_check_block);
|
|
|
|
CREATE_BLOCK(terminate_block, "terminate");
|
|
MOVE_BLOCK_AFTER_CURR(terminate_block);
|
|
|
|
if (!(flag =
|
|
LLVMBuildAnd(comp_ctx->builder, terminate_flags,
|
|
I32_ONE, "termination_flag"))) {
|
|
aot_set_last_error("llvm build AND failed");
|
|
return false;
|
|
}
|
|
|
|
BUILD_ICMP(LLVMIntSGT, flag, I32_ZERO, res, "need_terminate");
|
|
BUILD_COND_BR(res, terminate_block, non_terminate_block);
|
|
|
|
/* Move builder to terminate block */
|
|
SET_BUILDER_POS(terminate_block);
|
|
if (!aot_build_zero_function_ret(comp_ctx, aot_func_type)) {
|
|
goto fail;
|
|
}
|
|
|
|
/* Move builder to terminate block */
|
|
SET_BUILDER_POS(non_terminate_block);
|
|
return true;
|
|
|
|
fail:
|
|
return false;
|
|
}
|
|
#endif /* End of WASM_ENABLE_THREAD_MGR */
|
|
|
|
bool
|
|
aot_compile_op_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
|
uint32 br_depth, uint8 **p_frame_ip)
|
|
{
|
|
AOTBlock *block_dst;
|
|
LLVMValueRef value_ret, value_param;
|
|
LLVMBasicBlockRef next_llvm_end_block;
|
|
char name[32];
|
|
uint32 i, param_index, result_index;
|
|
|
|
#if WASM_ENABLE_THREAD_MGR != 0
|
|
/* Insert suspend check point */
|
|
if (comp_ctx->enable_thread_mgr) {
|
|
if (!check_suspend_flags(comp_ctx, func_ctx))
|
|
return false;
|
|
}
|
|
#endif
|
|
|
|
if (!(block_dst = get_target_block(func_ctx, br_depth))) {
|
|
return false;
|
|
}
|
|
|
|
if (block_dst->label_type == LABEL_TYPE_LOOP) {
|
|
/* Dest block is Loop block */
|
|
/* Handle Loop parameters */
|
|
for (i = 0; i < block_dst->param_count; i++) {
|
|
param_index = block_dst->param_count - 1 - i;
|
|
POP(value_param, block_dst->param_types[param_index]);
|
|
ADD_TO_PARAM_PHIS(block_dst, value_param, param_index);
|
|
}
|
|
BUILD_BR(block_dst->llvm_entry_block);
|
|
}
|
|
else {
|
|
/* Dest block is Block/If/Function block */
|
|
/* Create the end block */
|
|
if (!block_dst->llvm_end_block) {
|
|
format_block_name(name, sizeof(name),
|
|
block_dst->block_index, block_dst->label_type,
|
|
LABEL_END);
|
|
CREATE_BLOCK(block_dst->llvm_end_block, name);
|
|
if ((next_llvm_end_block = find_next_llvm_end_block(block_dst)))
|
|
MOVE_BLOCK_BEFORE(block_dst->llvm_end_block,
|
|
next_llvm_end_block);
|
|
}
|
|
|
|
block_dst->is_reachable = true;
|
|
|
|
/* Handle result values */
|
|
CREATE_RESULT_VALUE_PHIS(block_dst);
|
|
for (i = 0; i < block_dst->result_count; i++) {
|
|
result_index = block_dst->result_count - 1 - i;
|
|
POP(value_ret, block_dst->result_types[result_index]);
|
|
ADD_TO_RESULT_PHIS(block_dst, value_ret, result_index);
|
|
}
|
|
/* Jump to the end block */
|
|
BUILD_BR(block_dst->llvm_end_block);
|
|
}
|
|
|
|
return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip);
|
|
fail:
|
|
return false;
|
|
}
|
|
|
|
bool
|
|
aot_compile_op_br_if(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
|
uint32 br_depth, uint8 **p_frame_ip)
|
|
{
|
|
AOTBlock *block_dst;
|
|
LLVMValueRef value_cmp, value, *values = NULL;
|
|
LLVMBasicBlockRef llvm_else_block, next_llvm_end_block;
|
|
char name[32];
|
|
uint32 i, param_index, result_index;
|
|
uint64 size;
|
|
|
|
#if WASM_ENABLE_THREAD_MGR != 0
|
|
/* Insert suspend check point */
|
|
if (comp_ctx->enable_thread_mgr) {
|
|
if (!check_suspend_flags(comp_ctx, func_ctx))
|
|
return false;
|
|
}
|
|
#endif
|
|
|
|
POP_COND(value_cmp);
|
|
if (!LLVMIsConstant(value_cmp)) {
|
|
/* Compare value is not constant, create condition br IR */
|
|
if (!(block_dst = get_target_block(func_ctx, br_depth))) {
|
|
return false;
|
|
}
|
|
|
|
/* Create llvm else block */
|
|
CREATE_BLOCK(llvm_else_block, "br_if_else");
|
|
MOVE_BLOCK_AFTER_CURR(llvm_else_block);
|
|
|
|
if (block_dst->label_type == LABEL_TYPE_LOOP) {
|
|
/* Dest block is Loop block */
|
|
/* Handle Loop parameters */
|
|
if (block_dst->param_count) {
|
|
size = sizeof(LLVMValueRef) * (uint64)block_dst->param_count;
|
|
if (size >= UINT32_MAX
|
|
|| !(values = wasm_runtime_malloc((uint32)size))) {
|
|
aot_set_last_error("allocate memory failed.");
|
|
goto fail;
|
|
}
|
|
for (i = 0; i < block_dst->param_count; i++) {
|
|
param_index = block_dst->param_count - 1 - i;
|
|
POP(value, block_dst->param_types[param_index]);
|
|
ADD_TO_PARAM_PHIS(block_dst, value, param_index);
|
|
values[param_index] = value;
|
|
}
|
|
for (i = 0; i < block_dst->param_count; i++) {
|
|
PUSH(values[i], block_dst->param_types[i]);
|
|
}
|
|
wasm_runtime_free(values);
|
|
values = NULL;
|
|
}
|
|
|
|
BUILD_COND_BR(value_cmp, block_dst->llvm_entry_block,
|
|
llvm_else_block);
|
|
|
|
/* Move builder to else block */
|
|
SET_BUILDER_POS(llvm_else_block);
|
|
}
|
|
else {
|
|
/* Dest block is Block/If/Function block */
|
|
/* Create the end block */
|
|
if (!block_dst->llvm_end_block) {
|
|
format_block_name(name, sizeof(name),
|
|
block_dst->block_index, block_dst->label_type,
|
|
LABEL_END);
|
|
CREATE_BLOCK(block_dst->llvm_end_block, name);
|
|
if ((next_llvm_end_block = find_next_llvm_end_block(block_dst)))
|
|
MOVE_BLOCK_BEFORE(block_dst->llvm_end_block,
|
|
next_llvm_end_block);
|
|
}
|
|
|
|
/* Set reachable flag and create condition br IR */
|
|
block_dst->is_reachable = true;
|
|
|
|
/* Handle result values */
|
|
if (block_dst->result_count) {
|
|
size = sizeof(LLVMValueRef) * (uint64)block_dst->result_count;
|
|
if (size >= UINT32_MAX
|
|
|| !(values = wasm_runtime_malloc((uint32)size))) {
|
|
aot_set_last_error("allocate memory failed.");
|
|
goto fail;
|
|
}
|
|
CREATE_RESULT_VALUE_PHIS(block_dst);
|
|
for (i = 0; i < block_dst->result_count; i++) {
|
|
result_index = block_dst->result_count - 1 - i;
|
|
POP(value, block_dst->result_types[result_index]);
|
|
values[result_index] = value;
|
|
ADD_TO_RESULT_PHIS(block_dst, value, result_index);
|
|
}
|
|
for (i = 0; i < block_dst->result_count; i++) {
|
|
PUSH(values[i], block_dst->result_types[i]);
|
|
}
|
|
wasm_runtime_free(values);
|
|
values = NULL;
|
|
}
|
|
|
|
/* Condition jump to end block */
|
|
BUILD_COND_BR(value_cmp, block_dst->llvm_end_block,
|
|
llvm_else_block);
|
|
|
|
/* Move builder to else block */
|
|
SET_BUILDER_POS(llvm_else_block);
|
|
}
|
|
}
|
|
else {
|
|
if ((int32)LLVMConstIntGetZExtValue(value_cmp) != 0) {
|
|
/* Compare value is not 0, condition is true, same as op_br */
|
|
return aot_compile_op_br(comp_ctx, func_ctx, br_depth, p_frame_ip);
|
|
}
|
|
else {
|
|
/* Compare value is not 0, condition is false, skip br_if */
|
|
return true;
|
|
}
|
|
}
|
|
return true;
|
|
fail:
|
|
if (values)
|
|
wasm_runtime_free(values);
|
|
return false;
|
|
}
|
|
|
|
bool
|
|
aot_compile_op_br_table(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
|
uint32 *br_depths, uint32 br_count,
|
|
uint8 **p_frame_ip)
|
|
{
|
|
uint32 i, j;
|
|
LLVMValueRef value_switch, value_cmp, value_case, value, *values = NULL;
|
|
LLVMBasicBlockRef default_llvm_block = NULL, target_llvm_block;
|
|
LLVMBasicBlockRef next_llvm_end_block;
|
|
AOTBlock *target_block;
|
|
uint32 br_depth, depth_idx;
|
|
uint32 param_index, result_index;
|
|
uint64 size;
|
|
char name[32];
|
|
|
|
#if WASM_ENABLE_THREAD_MGR != 0
|
|
/* Insert suspend check point */
|
|
if (comp_ctx->enable_thread_mgr) {
|
|
if (!check_suspend_flags(comp_ctx, func_ctx))
|
|
return false;
|
|
}
|
|
#endif
|
|
|
|
POP_I32(value_cmp);
|
|
if (!LLVMIsConstant(value_cmp)) {
|
|
/* Compare value is not constant, create switch IR */
|
|
for (i = 0; i <= br_count; i++) {
|
|
target_block = get_target_block(func_ctx, br_depths[i]);
|
|
if (!target_block)
|
|
return false;
|
|
|
|
if (target_block->label_type != LABEL_TYPE_LOOP) {
|
|
/* Dest block is Block/If/Function block */
|
|
/* Create the end block */
|
|
if (!target_block->llvm_end_block) {
|
|
format_block_name(name, sizeof(name),
|
|
target_block->block_index,
|
|
target_block->label_type,
|
|
LABEL_END);
|
|
CREATE_BLOCK(target_block->llvm_end_block, name);
|
|
if ((next_llvm_end_block =
|
|
find_next_llvm_end_block(target_block)))
|
|
MOVE_BLOCK_BEFORE(target_block->llvm_end_block,
|
|
next_llvm_end_block);
|
|
}
|
|
/* Handle result values */
|
|
if (target_block->result_count) {
|
|
size = sizeof(LLVMValueRef) * (uint64)target_block->result_count;
|
|
if (size >= UINT32_MAX
|
|
|| !(values = wasm_runtime_malloc((uint32)size))) {
|
|
aot_set_last_error("allocate memory failed.");
|
|
goto fail;
|
|
}
|
|
CREATE_RESULT_VALUE_PHIS(target_block);
|
|
for (j = 0; j < target_block->result_count; j++) {
|
|
result_index = target_block->result_count - 1 - j;
|
|
POP(value, target_block->result_types[result_index]);
|
|
values[result_index] = value;
|
|
ADD_TO_RESULT_PHIS(target_block, value, result_index);
|
|
}
|
|
for (j = 0; j < target_block->result_count; j++) {
|
|
PUSH(values[j], target_block->result_types[j]);
|
|
}
|
|
wasm_runtime_free(values);
|
|
}
|
|
target_block->is_reachable = true;
|
|
if (i == br_count)
|
|
default_llvm_block = target_block->llvm_end_block;
|
|
}
|
|
else {
|
|
/* Handle Loop parameters */
|
|
if (target_block->param_count) {
|
|
size = sizeof(LLVMValueRef) * (uint64)target_block->param_count;
|
|
if (size >= UINT32_MAX
|
|
|| !(values = wasm_runtime_malloc((uint32)size))) {
|
|
aot_set_last_error("allocate memory failed.");
|
|
goto fail;
|
|
}
|
|
for (j = 0; j < target_block->param_count; j++) {
|
|
param_index = target_block->param_count - 1 - j;
|
|
POP(value, target_block->param_types[param_index]);
|
|
values[param_index] = value;
|
|
ADD_TO_PARAM_PHIS(target_block, value, param_index);
|
|
}
|
|
for (j = 0; j < target_block->param_count; j++) {
|
|
PUSH(values[j], target_block->param_types[j]);
|
|
}
|
|
wasm_runtime_free(values);
|
|
}
|
|
if (i == br_count)
|
|
default_llvm_block = target_block->llvm_entry_block;
|
|
}
|
|
}
|
|
|
|
/* Create switch IR */
|
|
if (!(value_switch = LLVMBuildSwitch(comp_ctx->builder, value_cmp,
|
|
default_llvm_block, br_count))) {
|
|
aot_set_last_error("llvm build switch failed.");
|
|
return false;
|
|
}
|
|
|
|
/* Add each case for switch IR */
|
|
for (i = 0; i < br_count; i++) {
|
|
value_case = I32_CONST(i);
|
|
CHECK_LLVM_CONST(value_case);
|
|
target_block = get_target_block(func_ctx, br_depths[i]);
|
|
if (!target_block)
|
|
return false;
|
|
target_llvm_block = target_block->label_type != LABEL_TYPE_LOOP
|
|
? target_block->llvm_end_block
|
|
: target_block->llvm_entry_block;
|
|
LLVMAddCase(value_switch, value_case, target_llvm_block);
|
|
}
|
|
|
|
return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip);
|
|
}
|
|
else {
|
|
/* Compare value is constant, create br IR */
|
|
depth_idx = (uint32)LLVMConstIntGetZExtValue(value_cmp);
|
|
br_depth = br_depths[br_count];
|
|
if (depth_idx < br_count) {
|
|
br_depth = br_depths[depth_idx];
|
|
}
|
|
return aot_compile_op_br(comp_ctx, func_ctx, br_depth, p_frame_ip);
|
|
}
|
|
fail:
|
|
if (values)
|
|
wasm_runtime_free(values);
|
|
return false;
|
|
}
|
|
|
|
bool
|
|
aot_compile_op_return(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
|
uint8 **p_frame_ip)
|
|
{
|
|
AOTBlock *block_func = func_ctx->block_stack.block_list_head;
|
|
LLVMValueRef value;
|
|
AOTFuncType *func_type;
|
|
uint32 i, param_index, result_index;
|
|
|
|
bh_assert(block_func);
|
|
func_type = func_ctx->aot_func->func_type;
|
|
|
|
if (block_func->result_count) {
|
|
/* Store extra result values to function parameters */
|
|
for (i = 0; i < block_func->result_count - 1; i++) {
|
|
result_index = block_func->result_count - 1 - i;
|
|
POP(value, block_func->result_types[result_index]);
|
|
param_index = func_type->param_count + result_index;
|
|
if (!LLVMBuildStore(comp_ctx->builder,
|
|
value,
|
|
LLVMGetParam(func_ctx->func, param_index))) {
|
|
aot_set_last_error("llvm build store failed.");
|
|
goto fail;
|
|
}
|
|
}
|
|
/* Return the first result value */
|
|
POP(value, block_func->result_types[0]);
|
|
if (!LLVMBuildRet(comp_ctx->builder, value)) {
|
|
aot_set_last_error("llvm build return failed.");
|
|
goto fail;
|
|
}
|
|
}
|
|
else {
|
|
if (!LLVMBuildRetVoid(comp_ctx->builder)) {
|
|
aot_set_last_error("llvm build return void failed.");
|
|
goto fail;
|
|
}
|
|
}
|
|
|
|
return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip);
|
|
fail:
|
|
return false;
|
|
}
|
|
|
|
bool
|
|
aot_compile_op_unreachable(AOTCompContext *comp_ctx,
|
|
AOTFuncContext *func_ctx,
|
|
uint8 **p_frame_ip)
|
|
{
|
|
if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_UNREACHABLE,
|
|
false, NULL, NULL))
|
|
return false;
|
|
|
|
return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip);
|
|
}
|
|
|
|
bool
|
|
aot_handle_next_reachable_block(AOTCompContext *comp_ctx,
|
|
AOTFuncContext *func_ctx,
|
|
uint8 **p_frame_ip)
|
|
{
|
|
return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip);
|
|
}
|