Update cmake files and wamr-test-suites to support collect code coverage (#1992)

Support collecting code coverage with wamr-test-suites script by using
lcov and genhtml tools, eg.:
  cd tests/wamr-test-suites
  ./test_wamr.sh -s spec -b -P -C

The default code coverage and html files are generated at:
  tests/wamr-test-suites/workspace/wamr.lcov
  tests/wamr-test-suites/workspace/wamr-lcov.zip

And update wamr-test-suites scripts to support testing GC spec cases to
avoid frequent synchronization conflicts between branch main and dev/gc.
This commit is contained in:
Wenyong Huang 2023-02-28 17:38:18 +08:00 committed by GitHub
parent b4f0228497
commit 9b9ae0cfac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 342 additions and 116 deletions

View File

@ -105,11 +105,6 @@ if (NOT DEFINED WAMR_BUILD_REF_TYPES)
set (WAMR_BUILD_REF_TYPES 0)
endif ()
if (COLLECT_CODE_COVERAGE EQUAL 1)
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
endif ()
set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR})
include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake)

View File

@ -341,3 +341,12 @@ if (WAMR_BUILD_WASM_CACHE EQUAL 1)
add_definitions (-DWASM_ENABLE_WASM_CACHE=1)
message (" Wasm files cache enabled")
endif ()
if (WAMR_BUILD_GC_HEAP_VERIFY EQUAL 1)
add_definitions (-DWASM_ENABLE_GC_VERIFY=1)
message (" GC heap verification enabled")
endif ()
if ("$ENV{COLLECT_CODE_COVERAGE}" STREQUAL "1" OR COLLECT_CODE_COVERAGE EQUAL 1)
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
message (" Collect code coverage enabled")
endif ()

View File

@ -110,11 +110,6 @@ if (WAMR_BUILD_DEBUG_INTERP EQUAL 1)
set (WAMR_BUILD_SIMD 0)
endif ()
if (COLLECT_CODE_COVERAGE EQUAL 1)
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
endif ()
set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../../../..)
include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake)

View File

@ -89,10 +89,6 @@ if (NOT DEFINED WAMR_BUILD_SGX_IPFS)
set (WAMR_BUILD_SGX_IPFS 0)
endif ()
if (COLLECT_CODE_COVERAGE EQUAL 1)
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
endif ()
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu11 -ffunction-sections -fdata-sections \
-Wall -Wno-unused-parameter -Wno-pedantic \

View File

@ -69,10 +69,6 @@ if (NOT DEFINED WAMR_BUILD_LIB_PTHREAD)
set (WAMR_BUILD_LIB_PTHREAD 0)
endif ()
if (COLLECT_CODE_COVERAGE EQUAL 1)
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
endif ()
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99 -ffunction-sections -fdata-sections \
-Wall -Wno-unused-parameter -Wno-pedantic \

View File

@ -117,11 +117,6 @@ if (WAMR_BUILD_DEBUG_INTERP EQUAL 1)
set (WAMR_BUILD_SIMD 0)
endif ()
if (COLLECT_CODE_COVERAGE EQUAL 1)
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
endif ()
set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..)
include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake)

View File

@ -96,10 +96,6 @@ if (WAMR_BUILD_DEBUG_INTERP EQUAL 1)
set (WAMR_BUILD_SIMD 0)
endif ()
if (COLLECT_CODE_COVERAGE EQUAL 1)
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
endif ()
set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..)
include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake)

View File

@ -39,10 +39,6 @@ set (WAMR_BUILD_FAST_INTERP 1)
set (WAMR_BUILD_LIB_RATS 1)
# compiling and linking flags
if (COLLECT_CODE_COVERAGE EQUAL 1)
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
endif ()
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu11 -ffunction-sections -fdata-sections \
-Wall -Wno-unused-parameter -Wno-pedantic \

View File

@ -16,6 +16,11 @@ set (CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang")
set (CMAKE_ASM_COMPILER "${WASI_SDK_DIR}/bin/clang")
set (CMAKE_EXE_LINKER_FLAGS "-target wasm32-wasi-threads")
if ("$ENV{COLLECT_CODE_COVERAGE}" STREQUAL "1" OR COLLECT_CODE_COVERAGE EQUAL 1)
set (CMAKE_C_FLAGS "")
set (CMAKE_CXX_FLAGS "")
endif ()
function (compile_sample SOURCE_FILE)
get_filename_component (FILE_NAME ${SOURCE_FILE} NAME_WLE)
set (WASM_MODULE ${FILE_NAME}.wasm)

View File

@ -14,6 +14,18 @@ import time
"""
The script itself has to be put under the same directory with the "spec".
To run a single non-GC case with interpreter mode:
cd workspace
python3 runtest.py --wast2wasm wabt/bin/wat2wasm --interpreter iwasm \
spec/test/core/xxx.wast
To run a single non-GC case with aot mode:
cd workspace
python3 runtest.py --aot --wast2wasm wabt/bin/wat2wasm --interpreter iwasm \
--aot-compiler wamrc spec/test/core/xxx.wast
To run a single GC case:
cd workspace
python3 runtest.py --wast2wasm spec/interpreter/wasm --interpreter iwasm \
--aot-compiler wamrc --gc spec/test/core/xxx.wast
"""
PLATFORM_NAME = os.uname().sysname.lower()
@ -22,9 +34,9 @@ IWASM_SGX_CMD = "../../../product-mini/platforms/linux-sgx/enclave-sample/iwasm"
IWASM_QEMU_CMD = "iwasm"
SPEC_TEST_DIR = "spec/test/core"
WAST2WASM_CMD = "./wabt/out/gcc/Release/wat2wasm"
SPEC_INTERPRETER_CMD = "spec/interpreter/wasm"
WAMRC_CMD = "../../../wamr-compiler/build/wamrc"
class TargetAction(argparse.Action):
TARGET_MAP = {
"ARMV7_VFP": "armv7",
@ -51,6 +63,7 @@ def ignore_the_case(
multi_module_flag=False,
multi_thread_flag=False,
simd_flag=False,
gc_flag=False,
xip_flag=False,
qemu_flag=False
):
@ -63,6 +76,10 @@ def ignore_the_case(
if "i386" == target and case_name in ["float_exprs"]:
return True
if gc_flag:
if case_name in ["type-canon", "type-equivalence", "type-rec"]:
return True;
if sgx_flag:
if case_name in ["conversions", "f32_bitwise", "f64_bitwise"]:
return True
@ -76,7 +93,9 @@ def ignore_the_case(
return True
if qemu_flag:
if case_name in ["f32_bitwise", "f64_bitwise", "loop", "f64", "f64_cmp", "conversions", "f32", "f32_cmp", "float_exprs", "float_misc", "select", "memory_grow"]:
if case_name in ["f32_bitwise", "f64_bitwise", "loop", "f64", "f64_cmp",
"conversions", "f32", "f32_cmp", "float_exprs",
"float_misc", "select", "memory_grow"]:
return True
return False
@ -109,6 +128,7 @@ def test_case(
xip_flag=False,
clean_up_flag=True,
verbose_flag=True,
gc_flag=False,
qemu_flag=False,
qemu_firmware='',
log='',
@ -124,6 +144,7 @@ def test_case(
multi_module_flag,
multi_thread_flag,
simd_flag,
gc_flag,
xip_flag,
qemu_flag
):
@ -131,7 +152,7 @@ def test_case(
CMD = ["python3", "runtest.py"]
CMD.append("--wast2wasm")
CMD.append(WAST2WASM_CMD)
CMD.append(WAST2WASM_CMD if not gc_flag else SPEC_INTERPRETER_CMD)
CMD.append("--interpreter")
if sgx_flag:
CMD.append(IWASM_SGX_CMD)
@ -171,6 +192,9 @@ def test_case(
if not clean_up_flag:
CMD.append("--no_cleanup")
if gc_flag:
CMD.append("--gc")
if log != '':
CMD.append("--log-dir")
CMD.append(log)
@ -231,6 +255,7 @@ def test_suite(
xip_flag=False,
clean_up_flag=True,
verbose_flag=True,
gc_flag=False,
parl_flag=False,
qemu_flag=False,
qemu_firmware='',
@ -246,6 +271,10 @@ def test_suite(
simd_case_list = sorted(suite_path.glob("simd/*.wast"))
case_list.extend(simd_case_list)
if gc_flag:
gc_case_list = sorted(suite_path.glob("gc/*.wast"))
case_list.extend(gc_case_list)
case_count = len(case_list)
failed_case = 0
successful_case = 0
@ -268,6 +297,7 @@ def test_suite(
xip_flag,
clean_up_flag,
verbose_flag,
gc_flag,
qemu_flag,
qemu_firmware,
log,
@ -304,6 +334,7 @@ def test_suite(
xip_flag,
clean_up_flag,
verbose_flag,
gc_flag,
qemu_flag,
qemu_firmware,
log,
@ -414,6 +445,13 @@ def main():
dest="verbose_flag",
help="Close real time output while running cases, only show last words of failed ones",
)
parser.add_argument(
"--gc",
action="store_true",
default=False,
dest="gc_flag",
help="Running with GC feature",
)
parser.add_argument(
"cases",
metavar="path_to__case",
@ -446,6 +484,7 @@ def main():
options.xip_flag,
options.clean_up_flag,
options.verbose_flag,
options.gc_flag,
options.parl_flag,
options.qemu_flag,
options.qemu_firmware,
@ -469,6 +508,7 @@ def main():
options.xip_flag,
options.clean_up_flag,
options.verbose_flag,
options.gc_flag,
options.qemu_flag,
options.qemu_firmware,
options.log

View File

@ -0,0 +1,80 @@
#!/usr/bin/env bash
#
# Copyright (C) 2019 Intel Corporation. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
readonly WORK_DIR=$PWD
readonly WAMR_DIR=${WORK_DIR}/../../..
readonly DST_COV_FILE=$1
readonly SRC_COV_DIR=$2
readonly SRC_TEMP_COV_FILE=wamr_temp.lcov
readonly SRC_COV_FILE=wamr.lcov
# get dest folder
dir=$(dirname ${DST_COV_FILE})
pushd ${dir} > /dev/null 2>&1
readonly DST_COV_DIR=${PWD}
popd > /dev/null 2>&1
if [[ ! -d ${SRC_COV_DIR} ]]; then
echo "${SRC_COV_DIR} doesn't exist, ignore code coverage collection"
exit
fi
echo "Start to collect code coverage of ${SRC_COV_DIR} .."
pushd ${SRC_COV_DIR} > /dev/null 2>&1
# collect all code coverage data
lcov -o ${SRC_TEMP_COV_FILE} -c -d . --rc lcov_branch_coverage=1
# extract code coverage data of WAMR source files
lcov -r ${SRC_TEMP_COV_FILE} -o ${SRC_TEMP_COV_FILE} \
-rc lcov_branch_coverage=1 \
"*/usr/*" "*/_deps/*" "*/deps/*" "*/tests/unit/*" \
"*/llvm/include/*" "*/include/llvm/*" "*/samples/*" \
"*/app-framework/*" "*/test-tools/*"
if [[ -s ${SRC_TEMP_COV_FILE} ]]; then
if [[ -s ${DST_COV_FILE} ]]; then
# merge code coverage data
lcov --rc lcov_branch_coverage=1 \
--add-tracefile ${SRC_TEMP_COV_FILE} \
-a ${DST_COV_FILE} -o ${SRC_COV_FILE}
# backup the original lcov file
cp -a ${DST_COV_FILE} "${DST_COV_FILE}.orig"
# replace the lcov file
cp -a ${SRC_COV_FILE} ${DST_COV_FILE}
else
cp -a ${SRC_TEMP_COV_FILE} ${SRC_COV_FILE}
cp -a ${SRC_COV_FILE} ${DST_COV_FILE}
fi
# get ignored prefix path
dir=$(dirname ${WAMR_DIR}/../..)
pushd ${dir} > /dev/null 2>&1
prefix_full_path=${PWD}
popd > /dev/null 2>&1
# generate html output for merged code coverage data
rm -fr ${DST_COV_DIR}/wamr-lcov
genhtml -t "WAMR Code Coverage" \
--rc lcov_branch_coverage=1 --prefix=${prefix_full_path} \
-o ${DST_COV_DIR}/wamr-lcov \
${DST_COV_FILE}
cd ${DST_COV_DIR}
rm -f wamr-lcov.zip
zip -r -q -o wamr-lcov.zip wamr-lcov
rm -fr wamr-lcov
echo "Code coverage file ${DST_COV_FILE} was generated or appended"
echo "Code coverage html ${DST_COV_DIR}/wamr-lcov.zip was generated"
else
echo "generate code coverage html failed"
fi
echo ""
popd > /dev/null 2>&1

View File

@ -230,6 +230,9 @@ parser.add_argument('--multi-module', default=False, action='store_true',
parser.add_argument('--multi-thread', default=False, action='store_true',
help="Enable Multi-thread")
parser.add_argument('--gc', default=False, action='store_true',
help='Test with GC')
parser.add_argument('--qemu', default=False, action='store_true',
help="Enable QEMU")
@ -420,11 +423,20 @@ def parse_simple_const_w_type(number, type):
number = float.fromhex(number) if '0x' in number else float(number)
return number, "{:.7g}:{}".format(number, type)
elif type == "ref.null":
# hard coding
if number == "func":
return "func", "func:ref.null"
elif number == "extern":
return "extern", "extern:ref.null"
elif number == "any":
return "any", "any:ref.null"
else:
raise Exception("invalid value {} and type {}".format(number, type))
elif type == "ref.extern":
number = int(number, 16) if '0x' in number else int(number)
return number, "0x{:x}:ref.extern".format(number)
elif type == "ref.host":
number = int(number, 16) if '0x' in number else int(number)
return number, "0x{:x}:ref.host".format(number)
else:
raise Exception("invalid value {} and type {}".format(number, type))
@ -440,6 +452,10 @@ def parse_assertion_value(val):
type.const val
ref.extern val
ref.null ref_type
ref.array
ref.struct
ref.func
ref.i31
"""
if not val:
return None, ""
@ -453,6 +469,8 @@ def parse_assertion_value(val):
if type in ["i32", "i64", "f32", "f64"]:
return parse_simple_const_w_type(numbers[0], type)
elif type == "ref":
if splitted[0] in ["ref.array", "ref.struct", "ref.func", "ref.i31"]:
return splitted[0]
# need to distinguish between "ref.null" and "ref.extern"
return parse_simple_const_w_type(numbers[0], splitted[0])
else:
@ -615,6 +633,9 @@ def simple_value_comparison(out, expected):
elif "ref.extern" == expected_type:
out_val_binary = out_val
expected_val_binary = expected_val
elif "ref.host" == expected_type:
out_val_binary = out_val
expected_val_binary = expected_val
else:
assert(0), "unknown 'expected_type' {}".format(expected_type)
@ -637,7 +658,9 @@ def value_comparison(out, expected):
if not expected:
return False
if not out in ["ref.array", "ref.struct", "ref.func", "ref.any", "ref.i31"]:
assert(':' in out), "out should be in a form likes numbers:type, but {}".format(out)
if not expected in ["ref.array", "ref.struct", "ref.func", "ref.any", "ref.i31"]:
assert(':' in expected), "expected should be in a form likes numbers:type, but {}".format(expected)
if 'v128' in out:
@ -761,6 +784,9 @@ def test_assert_return(r, opts, form):
elif "ref.extern" == splitted[0]:
number, _ = parse_simple_const_w_type(splitted[1], splitted[0])
args.append(str(number))
elif "ref.host" == splitted[0]:
number, _ = parse_simple_const_w_type(splitted[1], splitted[0])
args.append(str(number))
else:
assert(0), "an unkonwn parameter type"
@ -769,6 +795,14 @@ def test_assert_return(r, opts, form):
else:
returns = re.split("\)\s*\(", m.group(3)[1:-1])
# processed numbers in strings
if len(returns) == 1 and returns[0] in ["ref.array", "ref.struct", "ref.i31",
"ref.eq", "ref.any", "ref.extern",
"ref.func", "ref.null"]:
expected = [returns[0]]
elif len(returns) == 1 and returns[0] in ["func:ref.null", "any:ref.null",
"extern:ref.null"]:
expected = [returns[0]]
else:
expected = [parse_assertion_value(v)[1] for v in returns]
expected = ",".join(expected)
@ -800,10 +834,10 @@ def test_assert_return(r, opts, form):
if n.group(3) == '':
args=[]
else:
args = [re.split(' +', v)[1] for v in re.split("\)\s*\(", n.group(3)[1:-1])]
# a workaround for "ref.null extern" and "ref.null func"
args = [ arg.replace('extern', 'null').replace('func', 'null') for arg in args]
# convert (ref.null extern/func) into (ref.null null)
n1 = n.group(3).replace("(ref.null extern)", "(ref.null null)")
n1 = n1.replace("ref.null func)", "(ref.null null)")
args = [re.split(' +', v)[1] for v in re.split("\)\s*\(", n1[1:-1])]
_, expected = parse_assertion_value(n.group(4)[1:-1])
test_assert(r, opts, "return", "%s %s" % (func, " ".join(args)), expected)
@ -828,10 +862,10 @@ def test_assert_trap(r, opts, form):
if m.group(2) == '':
args = []
else:
args = [re.split(' +', v)[1] for v in re.split("\)\s*\(", m.group(2)[1:-1])]
# workaround for "ref.null extern"
args = [ arg.replace('extern', 'null').replace('func', 'null') for arg in args]
# convert (ref.null extern/func) into (ref.null null)
m1 = m.group(2).replace("(ref.null extern)", "(ref.null null)")
m1 = m1.replace("ref.null func)", "(ref.null null)")
args = [re.split(' +', v)[1] for v in re.split("\)\s*\(", m1[1:-1])]
expected = "Exception: %s" % m.group(3)
test_assert(r, opts, "trap", "%s %s" % (func, " ".join(args)), expected)
@ -918,9 +952,10 @@ def compile_wast_to_wasm(form, wast_tempfile, wasm_tempfile, opts):
log("Compiling WASM to '%s'" % wasm_tempfile)
# default arguments
cmd = [opts.wast2wasm,
"--enable-thread",
"--no-check",
if opts.gc:
cmd = [opts.wast2wasm, "-u", "-d", wast_tempfile, "-o", wasm_tempfile]
else:
cmd = [opts.wast2wasm, "--enable-thread", "--no-check",
wast_tempfile, "-o", wasm_tempfile ]
# remove reference-type and bulk-memory enabling options since a WABT

View File

@ -14,19 +14,23 @@ function help()
{
echo "test_wamr.sh [options]"
echo "-c clean previous test results, not start test"
echo "-s {suite_name} test only one suite (spec)"
echo "-m set compile target of iwasm(x86_64\x86_32\armv7_vfp\thumbv7_vfp\riscv64_lp64d\riscv64_lp64)"
echo "-t set compile type of iwasm(classic-interp\fast-interp\jit\aot\fast-jit\multi-tier-jit)"
echo "-s {suite_name} test only one suite (spec|wasi_certification)"
echo "-m set compile target of iwasm(x86_64|x86_32|armv7_vfp|thumbv7_vfp|riscv64_lp64d|riscv64_lp64)"
echo "-t set compile type of iwasm(classic-interp|fast-interp|jit|aot|fast-jit|multi-tier-jit)"
echo "-M enable multi module feature"
echo "-p enable multi thread feature"
echo "-S enable SIMD feature"
echo "-G enable GC feature"
echo "-X enable XIP feature"
echo "-x test SGX"
echo "-w enable WASI threads"
echo "-b use the wabt binary release package instead of compiling from the source code"
echo "-g build iwasm with debug version"
echo "-v enable GC heap verification"
echo "-P run the spec test parallelly"
echo "-Q enable qemu"
echo "-F set the firmware path used by qemu"
echo "-w enable WASI threads"
echo "-C enable code coverage collect"
}
OPT_PARSED=""
@ -40,7 +44,10 @@ ENABLE_MULTI_MODULE=0
ENABLE_MULTI_THREAD=0
COLLECT_CODE_COVERAGE=0
ENABLE_SIMD=0
ENABLE_GC=0
ENABLE_XIP=0
ENABLE_DEBUG_VERSION=0
ENABLE_GC_HEAP_VERIFY=0
#unit test case arrary
TEST_CASE_ARR=()
SGX_OPT=""
@ -50,7 +57,7 @@ ENABLE_QEMU=0
QEMU_FIRMWARE=""
WASI_TESTSUITE_COMMIT="b18247e2161bea263fe924b8189c67b1d2d10a10"
while getopts ":s:cabt:m:wMCpSXxPQF:" opt
while getopts ":s:cabgvt:m:MCpSXxwPGQF:" opt
do
OPT_PARSED="TRUE"
case $opt in
@ -72,8 +79,9 @@ do
c)
read -t 5 -p "Are you sure to delete all reports. y/n " cmd
if [[ $cmd == "y" && $(ls -A workspace/report) ]];then
rm -r workspace/report/*
echo "cleaned all reports"
rm -fr workspace/report/*
rm -fr /tmp/*.wasm /tmp/*.wast /tmp/*.aot
echo "cleaned all reports and temp files"
fi
exit 0;;
a)
@ -128,6 +136,18 @@ do
echo "test SGX"
SGX_OPT="--sgx"
;;
g)
echo "enable build iwasm with debug version"
ENABLE_DEBUG_VERSION=1
;;
v)
echo "enable GC heap verification"
ENABLE_GC_HEAP_VERIFY=1
;;
G)
echo "enable GC feature"
ENABLE_GC=1
;;
P)
PARALLELISM=1
;;
@ -164,7 +184,6 @@ readonly DATE=$(date +%Y-%m-%d_%H:%M:%S)
readonly REPORT_DIR=${WORK_DIR}/report/${DATE}
mkdir -p ${REPORT_DIR}
# TODO: a strong assumation about a link to the WAMR project
readonly WAMR_DIR=${WORK_DIR}/../../..
if [[ ${SGX_OPT} == "--sgx" ]];then
@ -198,14 +217,16 @@ readonly ORC_EAGER_JIT_COMPILE_FLAGS="\
-DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_INTERP=0 \
-DWAMR_BUILD_JIT=1 -DWAMR_BUILD_AOT=1 \
-DWAMR_BUILD_LAZY_JIT=0 \
-DWAMR_BUILD_SPEC_TEST=1"
-DWAMR_BUILD_SPEC_TEST=1 \
-DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}"
readonly ORC_LAZY_JIT_COMPILE_FLAGS="\
-DWAMR_BUILD_TARGET=${TARGET} \
-DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_INTERP=0 \
-DWAMR_BUILD_JIT=1 -DWAMR_BUILD_AOT=1 \
-DWAMR_BUILD_LAZY_JIT=1 \
-DWAMR_BUILD_SPEC_TEST=1"
-DWAMR_BUILD_SPEC_TEST=1 \
-DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}"
readonly AOT_COMPILE_FLAGS="\
-DWAMR_BUILD_TARGET=${TARGET} \
@ -219,13 +240,15 @@ readonly FAST_JIT_COMPILE_FLAGS="\
-DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_INTERP=0 \
-DWAMR_BUILD_JIT=0 -DWAMR_BUILD_AOT=0 \
-DWAMR_BUILD_FAST_JIT=1 \
-DWAMR_BUILD_SPEC_TEST=1"
-DWAMR_BUILD_SPEC_TEST=1 \
-DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}"
readonly MULTI_TIER_JIT_COMPILE_FLAGS="\
-DWAMR_BUILD_TARGET=${TARGET} \
-DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_INTERP=0 \
-DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_JIT=1 \
-DWAMR_BUILD_SPEC_TEST=1"
-DWAMR_BUILD_SPEC_TEST=1 \
-DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}"
readonly COMPILE_FLAGS=(
"${CLASSIC_INTERP_COMPILE_FLAGS}"
@ -237,39 +260,19 @@ readonly COMPILE_FLAGS=(
"${MULTI_TIER_JIT_COMPILE_FLAGS}"
)
# TODO: with libiwasm.so only
function unit_test()
{
echo "Now start unit tests"
cd ${WORK_DIR}
readonly UNIT_CASES="wasm-vm host-tool utils"
rm -fr unittest-build && mkdir unittest-build
cd unittest-build
echo "Build unit test"
touch ${REPORT_DIR}/unit_test_report.txt
for compile_flag in "${COMPILE_FLAGS[@]}"; do
echo "Build unit test with compile flags with " ${compile_flag}
# keep going and do not care if it is success or not
make -ki clean | true
cmake ${compile_flag} ${WORK_DIR}/../../unit && make -j 4
if [ "$?" != 0 ];then
echo -e "build unit test failed, you may need to change wamr into dev/aot branch and ensure llvm is built"
exit 1
fi
echo ${compile_flag} >> ${REPORT_DIR}/unit_test_report.txt
for case in ${UNIT_CASES}
do
echo "run ${case} ..."
cd ./${case}/
./${case/-/_}"_test" | tee -a ${REPORT_DIR}/unit_test_report.txt
cd -
echo "finish ${case}"
done
done
cmake ${WORK_DIR}/../../unit -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}
make -j
make test | tee -a ${REPORT_DIR}/unit_test_report.txt
echo "Finish unit tests"
}
@ -347,6 +350,27 @@ function spec_test()
git apply ../../spec-test-script/thread_proposal_fix_atomic_case.patch
fi
# update GC cases
if [[ ${ENABLE_GC} == 1 ]]; then
echo "checkout spec for GC proposal"
popd
rm -fr spec
# check spec test cases for GC
git clone -b main --single-branch https://github.com/WebAssembly/gc.git spec
pushd spec
git restore . && git clean -ffd .
# Sync constant expression descriptions
git reset --hard 62beb94ddd41987517781732f17f213d8b866dcc
git apply ../../spec-test-script/gc_ignore_cases.patch
echo "compile the reference intepreter"
pushd interpreter
make opt
popd
fi
popd
echo $(pwd)
@ -446,6 +470,10 @@ function spec_test()
ARGS_FOR_SPEC_TEST+="--parl "
fi
if [[ ${ENABLE_GC} == 1 ]]; then
ARGS_FOR_SPEC_TEST+="--gc "
fi
if [[ ${ENABLE_QEMU} == 1 ]]; then
ARGS_FOR_SPEC_TEST+="--qemu "
ARGS_FOR_SPEC_TEST+="--qemu-firmware ${QEMU_FIRMWARE} "
@ -484,7 +512,7 @@ function wasi_test()
function wasi_certification_test()
{
echo "Now start wasi tests"
echo "Now start wasi certification tests"
cd ${WORK_DIR}
if [ ! -d "wasi-testsuite" ]; then
@ -514,7 +542,6 @@ function polybench_test()
if [[ $1 == "aot" || $1 == "jit" ]];then
./build.sh AOT ${SGX_OPT}
./test_aot.sh $1 ${SGX_OPT}
else
./build.sh
./test_interp.sh ${SGX_OPT}
@ -524,6 +551,22 @@ function polybench_test()
echo "Finish polybench tests"
}
function libsodium_test()
{
echo "Now start libsodium tests"
cd ${WORK_DIR}/../libsodium
if [[ $1 == "aot" || $1 == "jit" ]];then
./build.sh ${SGX_OPT}
./test_aot.sh $1 ${SGX_OPT}
else
./test_interp.sh ${SGX_OPT}
fi
cp report.txt ${REPORT_DIR}/libsodium_$1_test_report.txt
echo "Finish libsodium tests"
}
function malformed_test()
{
# build iwasm firstly
@ -535,14 +578,13 @@ function standalone_test()
{
cd ${WORK_DIR}/../../standalone
args=""
args="--$1"
[[ $1 == "aot" ]] && args="$args --aot" || args="$args --no-aot"
[[ ${SGX_OPT} == "--sgx" ]] && args="$args --sgx" || args="$args --no-sgx"
if [[ ${ENABLE_MULTI_THREAD} == 1 ]];then
args="$args --thread"
fi
[[ ${ENABLE_MULTI_THREAD} == 1 ]] && args="$args --thread" && args="$args --no-thread"
[[ ${ENABLE_SIMD} == 1 ]] && args="$args --simd" && args="$args --no-simd"
./standalone.sh $args | tee ${REPORT_DIR}/standalone_$1_test_report.txt
}
@ -589,7 +631,7 @@ function build_wamrc()
&& ./build_llvm.sh \
&& if [ -d build ]; then rm -r build/*; else mkdir build; fi \
&& cd build \
&& cmake .. \
&& cmake .. -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE} \
&& make -j 4
}
@ -603,14 +645,32 @@ function build_wamrc()
function collect_coverage()
{
if [[ ${COLLECT_CODE_COVERAGE} == 1 ]]; then
cd ${IWASM_LINUX_ROOT_DIR}/build
lcov -t "iwasm code coverage" -o iwasm.info -c -d .
genhtml -o iwasm-gcov iwasm.info
[[ -d iwasm-gcov ]] && \
cp -r iwasm-gcov ${REPORT_DIR}/$1_iwasm_gcov || \
echo "generate code coverage html failed"
ln -sf ${WORK_DIR}/../spec-test-script/collect_coverage.sh ${WORK_DIR}
CODE_COV_FILE=""
if [[ -z "${COV_FILE}" ]]; then
CODE_COV_FILE="${WORK_DIR}/wamr.lcov"
else
echo "will not collect code coverage"
CODE_COV_FILE="${COV_FILE}"
fi
pushd ${WORK_DIR} > /dev/null 2>&1
echo "Collect code coverage of iwasm"
./collect_coverage.sh ${CODE_COV_FILE} ${IWASM_LINUX_ROOT_DIR}/build
if [[ $1 == "llvm-aot" ]]; then
echo "Collect code coverage of wamrc"
./collect_coverage.sh ${CODE_COV_FILE} ${WAMR_DIR}/wamr-compiler/build
fi
for suite in "${TEST_CASE_ARR[@]}"; do
if [[ ${suite} = "unit" ]]; then
echo "Collect code coverage of unit test"
./collect_coverage.sh ${CODE_COV_FILE} ${WORK_DIR}/unittest-build
break
fi
done
popd > /dev/null 2>&1
else
echo "code coverage isn't collected"
fi
}
@ -639,10 +699,22 @@ function trigger()
EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_SIMD=0"
fi
if [[ ${ENABLE_GC} == 1 ]]; then
EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_GC=1"
EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_REF_TYPES=1"
EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_BULK_MEMORY=1"
fi
if [[ ${ENABLE_DEBUG_VERSION} == 1 ]]; then
EXTRA_COMPILE_FLAGS+=" -DCMAKE_BUILD_TYPE=Debug"
fi
if [[ ${ENABLE_GC_HEAP_VERIFY} == 1 ]]; then
EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_GC_HEAP_VERIFY=1"
fi
if [[ ${ENABLE_WASI_THREADS} == 1 ]]; then
EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_LIB_WASI_THREADS=1"
else
EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_LIB_WASI_THREADS=0"
fi
for t in "${TYPE[@]}"; do
@ -692,18 +764,18 @@ function trigger()
echo "work in orc jit eager compilation mode"
BUILD_FLAGS="$ORC_EAGER_JIT_COMPILE_FLAGS $EXTRA_COMPILE_FLAGS"
build_iwasm_with_cfg $BUILD_FLAGS
build_wamrc
for suite in "${TEST_CASE_ARR[@]}"; do
$suite"_test" jit
done
collect_coverage llvm-jit
echo "work in orc jit lazy compilation mode"
BUILD_FLAGS="$ORC_EAGER_JIT_COMPILE_FLAGS $EXTRA_COMPILE_FLAGS"
build_iwasm_with_cfg $BUILD_FLAGS
build_wamrc
for suite in "${TEST_CASE_ARR[@]}"; do
$suite"_test" jit
done
collect_coverage llvm-jit
;;
"aot")
@ -717,7 +789,7 @@ function trigger()
for suite in "${TEST_CASE_ARR[@]}"; do
$suite"_test" aot
done
collect_coverage aot
collect_coverage llvm-aot
;;
"fast-jit")
@ -728,6 +800,7 @@ function trigger()
for suite in "${TEST_CASE_ARR[@]}"; do
$suite"_test" fast-jit
done
collect_coverage fast-jit
;;
"multi-tier-jit")
@ -738,6 +811,7 @@ function trigger()
for suite in "${TEST_CASE_ARR[@]}"; do
$suite"_test" multi-tier-jit
done
collect_coverage multi-tier-jit
;;
*)
@ -748,11 +822,19 @@ function trigger()
}
# if collect code coverage, ignore -s, test all test cases.
if [[ $TEST_CASE_ARR && $COLLECT_CODE_COVERAGE != 1 ]];then
if [[ $TEST_CASE_ARR ]];then
trigger || (echo "TEST FAILED"; exit 1)
else
# test all suite, ignore polybench because of long time cost
TEST_CASE_ARR=("sightglass" "spec" "wasi" "malformed" "standalone")
# test all suite, ignore polybench and libsodium because of long time cost
TEST_CASE_ARR=("spec")
: '
if [[ $COLLECT_CODE_COVERAGE == 1 ]];then
# add polybench if collecting code coverage data
TEST_CASE_ARR+=("polybench")
# add libsodium if needed, which takes long time to run
TEST_CASE_ARR+=("libsodium")
fi
'
trigger || (echo "TEST FAILED"; exit 1)
# Add more suites here
fi

View File

@ -166,6 +166,12 @@ if (WAMR_BUILD_DEBUG_AOT EQUAL 1)
message(STATUS "find lldb ${LLDB_ALL_PLUGINS} in: ${LLVM_LIBRARY_DIRS}")
endif()
if ("$ENV{COLLECT_CODE_COVERAGE}" STREQUAL "1" OR COLLECT_CODE_COVERAGE EQUAL 1)
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
message ("-- Collect code coverage enabled")
endif ()
if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang"))
if(NOT MSVC)
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections")