From 9b9ae0cfac671195efa1a8c290a02318ae361127 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Tue, 28 Feb 2023 17:38:18 +0800 Subject: [PATCH] Update cmake files and wamr-test-suites to support collect code coverage (#1992) Support collecting code coverage with wamr-test-suites script by using lcov and genhtml tools, eg.: cd tests/wamr-test-suites ./test_wamr.sh -s spec -b -P -C The default code coverage and html files are generated at: tests/wamr-test-suites/workspace/wamr.lcov tests/wamr-test-suites/workspace/wamr-lcov.zip And update wamr-test-suites scripts to support testing GC spec cases to avoid frequent synchronization conflicts between branch main and dev/gc. --- CMakeLists.txt | 5 - build-scripts/config_common.cmake | 9 + .../libraries/wasi-nn/test/CMakeLists.txt | 5 - .../platforms/linux-sgx/CMakeLists.txt | 4 - .../linux-sgx/CMakeLists_minimal.txt | 4 - product-mini/platforms/linux/CMakeLists.txt | 5 - product-mini/platforms/windows/CMakeLists.txt | 4 - samples/sgx-ra/CMakeLists.txt | 4 - samples/wasi-threads/wasm-apps/CMakeLists.txt | 5 + .../wamr-test-suites/spec-test-script/all.py | 46 +++- .../spec-test-script/collect_coverage.sh | 80 +++++++ .../spec-test-script/runtest.py | 77 +++++-- tests/wamr-test-suites/test_wamr.sh | 204 ++++++++++++------ wamr-compiler/CMakeLists.txt | 6 + 14 files changed, 342 insertions(+), 116 deletions(-) create mode 100755 tests/wamr-test-suites/spec-test-script/collect_coverage.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 544922a1..b80e3bde 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -105,11 +105,6 @@ if (NOT DEFINED WAMR_BUILD_REF_TYPES) set (WAMR_BUILD_REF_TYPES 0) endif () -if (COLLECT_CODE_COVERAGE EQUAL 1) - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage") - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage") -endif () - set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}) include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index 53b0207f..3ac60846 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -341,3 +341,12 @@ if (WAMR_BUILD_WASM_CACHE EQUAL 1) add_definitions (-DWASM_ENABLE_WASM_CACHE=1) message (" Wasm files cache enabled") endif () +if (WAMR_BUILD_GC_HEAP_VERIFY EQUAL 1) + add_definitions (-DWASM_ENABLE_GC_VERIFY=1) + message (" GC heap verification enabled") +endif () +if ("$ENV{COLLECT_CODE_COVERAGE}" STREQUAL "1" OR COLLECT_CODE_COVERAGE EQUAL 1) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage") + message (" Collect code coverage enabled") +endif () diff --git a/core/iwasm/libraries/wasi-nn/test/CMakeLists.txt b/core/iwasm/libraries/wasi-nn/test/CMakeLists.txt index eff716bd..33fad71e 100644 --- a/core/iwasm/libraries/wasi-nn/test/CMakeLists.txt +++ b/core/iwasm/libraries/wasi-nn/test/CMakeLists.txt @@ -110,11 +110,6 @@ if (WAMR_BUILD_DEBUG_INTERP EQUAL 1) set (WAMR_BUILD_SIMD 0) endif () -if (COLLECT_CODE_COVERAGE EQUAL 1) - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage") - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage") -endif () - set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../../../..) include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) diff --git a/product-mini/platforms/linux-sgx/CMakeLists.txt b/product-mini/platforms/linux-sgx/CMakeLists.txt index 306f634e..e1cbe2cc 100644 --- a/product-mini/platforms/linux-sgx/CMakeLists.txt +++ b/product-mini/platforms/linux-sgx/CMakeLists.txt @@ -89,10 +89,6 @@ if (NOT DEFINED WAMR_BUILD_SGX_IPFS) set (WAMR_BUILD_SGX_IPFS 0) endif () -if (COLLECT_CODE_COVERAGE EQUAL 1) - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage") -endif () - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu11 -ffunction-sections -fdata-sections \ -Wall -Wno-unused-parameter -Wno-pedantic \ diff --git a/product-mini/platforms/linux-sgx/CMakeLists_minimal.txt b/product-mini/platforms/linux-sgx/CMakeLists_minimal.txt index b79565f1..aa3de6da 100644 --- a/product-mini/platforms/linux-sgx/CMakeLists_minimal.txt +++ b/product-mini/platforms/linux-sgx/CMakeLists_minimal.txt @@ -69,10 +69,6 @@ if (NOT DEFINED WAMR_BUILD_LIB_PTHREAD) set (WAMR_BUILD_LIB_PTHREAD 0) endif () -if (COLLECT_CODE_COVERAGE EQUAL 1) - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage") -endif () - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99 -ffunction-sections -fdata-sections \ -Wall -Wno-unused-parameter -Wno-pedantic \ diff --git a/product-mini/platforms/linux/CMakeLists.txt b/product-mini/platforms/linux/CMakeLists.txt index 7f846308..cc7ff8de 100644 --- a/product-mini/platforms/linux/CMakeLists.txt +++ b/product-mini/platforms/linux/CMakeLists.txt @@ -117,11 +117,6 @@ if (WAMR_BUILD_DEBUG_INTERP EQUAL 1) set (WAMR_BUILD_SIMD 0) endif () -if (COLLECT_CODE_COVERAGE EQUAL 1) - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage") - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage") -endif () - set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..) include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) diff --git a/product-mini/platforms/windows/CMakeLists.txt b/product-mini/platforms/windows/CMakeLists.txt index 58e5f384..35b22a60 100644 --- a/product-mini/platforms/windows/CMakeLists.txt +++ b/product-mini/platforms/windows/CMakeLists.txt @@ -96,10 +96,6 @@ if (WAMR_BUILD_DEBUG_INTERP EQUAL 1) set (WAMR_BUILD_SIMD 0) endif () -if (COLLECT_CODE_COVERAGE EQUAL 1) - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage") -endif () - set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..) include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) diff --git a/samples/sgx-ra/CMakeLists.txt b/samples/sgx-ra/CMakeLists.txt index 84e4d44c..7ab55224 100644 --- a/samples/sgx-ra/CMakeLists.txt +++ b/samples/sgx-ra/CMakeLists.txt @@ -39,10 +39,6 @@ set (WAMR_BUILD_FAST_INTERP 1) set (WAMR_BUILD_LIB_RATS 1) # compiling and linking flags -if (COLLECT_CODE_COVERAGE EQUAL 1) - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage") -endif () - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu11 -ffunction-sections -fdata-sections \ -Wall -Wno-unused-parameter -Wno-pedantic \ diff --git a/samples/wasi-threads/wasm-apps/CMakeLists.txt b/samples/wasi-threads/wasm-apps/CMakeLists.txt index c5869786..e77e5330 100644 --- a/samples/wasi-threads/wasm-apps/CMakeLists.txt +++ b/samples/wasi-threads/wasm-apps/CMakeLists.txt @@ -16,6 +16,11 @@ set (CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") set (CMAKE_ASM_COMPILER "${WASI_SDK_DIR}/bin/clang") set (CMAKE_EXE_LINKER_FLAGS "-target wasm32-wasi-threads") +if ("$ENV{COLLECT_CODE_COVERAGE}" STREQUAL "1" OR COLLECT_CODE_COVERAGE EQUAL 1) + set (CMAKE_C_FLAGS "") + set (CMAKE_CXX_FLAGS "") +endif () + function (compile_sample SOURCE_FILE) get_filename_component (FILE_NAME ${SOURCE_FILE} NAME_WLE) set (WASM_MODULE ${FILE_NAME}.wasm) diff --git a/tests/wamr-test-suites/spec-test-script/all.py b/tests/wamr-test-suites/spec-test-script/all.py index 7b29bcba..8b26d689 100644 --- a/tests/wamr-test-suites/spec-test-script/all.py +++ b/tests/wamr-test-suites/spec-test-script/all.py @@ -14,6 +14,18 @@ import time """ The script itself has to be put under the same directory with the "spec". +To run a single non-GC case with interpreter mode: + cd workspace + python3 runtest.py --wast2wasm wabt/bin/wat2wasm --interpreter iwasm \ + spec/test/core/xxx.wast +To run a single non-GC case with aot mode: + cd workspace + python3 runtest.py --aot --wast2wasm wabt/bin/wat2wasm --interpreter iwasm \ + --aot-compiler wamrc spec/test/core/xxx.wast +To run a single GC case: + cd workspace + python3 runtest.py --wast2wasm spec/interpreter/wasm --interpreter iwasm \ + --aot-compiler wamrc --gc spec/test/core/xxx.wast """ PLATFORM_NAME = os.uname().sysname.lower() @@ -22,9 +34,9 @@ IWASM_SGX_CMD = "../../../product-mini/platforms/linux-sgx/enclave-sample/iwasm" IWASM_QEMU_CMD = "iwasm" SPEC_TEST_DIR = "spec/test/core" WAST2WASM_CMD = "./wabt/out/gcc/Release/wat2wasm" +SPEC_INTERPRETER_CMD = "spec/interpreter/wasm" WAMRC_CMD = "../../../wamr-compiler/build/wamrc" - class TargetAction(argparse.Action): TARGET_MAP = { "ARMV7_VFP": "armv7", @@ -51,6 +63,7 @@ def ignore_the_case( multi_module_flag=False, multi_thread_flag=False, simd_flag=False, + gc_flag=False, xip_flag=False, qemu_flag=False ): @@ -63,6 +76,10 @@ def ignore_the_case( if "i386" == target and case_name in ["float_exprs"]: return True + if gc_flag: + if case_name in ["type-canon", "type-equivalence", "type-rec"]: + return True; + if sgx_flag: if case_name in ["conversions", "f32_bitwise", "f64_bitwise"]: return True @@ -76,7 +93,9 @@ def ignore_the_case( return True if qemu_flag: - if case_name in ["f32_bitwise", "f64_bitwise", "loop", "f64", "f64_cmp", "conversions", "f32", "f32_cmp", "float_exprs", "float_misc", "select", "memory_grow"]: + if case_name in ["f32_bitwise", "f64_bitwise", "loop", "f64", "f64_cmp", + "conversions", "f32", "f32_cmp", "float_exprs", + "float_misc", "select", "memory_grow"]: return True return False @@ -109,6 +128,7 @@ def test_case( xip_flag=False, clean_up_flag=True, verbose_flag=True, + gc_flag=False, qemu_flag=False, qemu_firmware='', log='', @@ -124,6 +144,7 @@ def test_case( multi_module_flag, multi_thread_flag, simd_flag, + gc_flag, xip_flag, qemu_flag ): @@ -131,7 +152,7 @@ def test_case( CMD = ["python3", "runtest.py"] CMD.append("--wast2wasm") - CMD.append(WAST2WASM_CMD) + CMD.append(WAST2WASM_CMD if not gc_flag else SPEC_INTERPRETER_CMD) CMD.append("--interpreter") if sgx_flag: CMD.append(IWASM_SGX_CMD) @@ -171,6 +192,9 @@ def test_case( if not clean_up_flag: CMD.append("--no_cleanup") + if gc_flag: + CMD.append("--gc") + if log != '': CMD.append("--log-dir") CMD.append(log) @@ -231,6 +255,7 @@ def test_suite( xip_flag=False, clean_up_flag=True, verbose_flag=True, + gc_flag=False, parl_flag=False, qemu_flag=False, qemu_firmware='', @@ -246,6 +271,10 @@ def test_suite( simd_case_list = sorted(suite_path.glob("simd/*.wast")) case_list.extend(simd_case_list) + if gc_flag: + gc_case_list = sorted(suite_path.glob("gc/*.wast")) + case_list.extend(gc_case_list) + case_count = len(case_list) failed_case = 0 successful_case = 0 @@ -268,6 +297,7 @@ def test_suite( xip_flag, clean_up_flag, verbose_flag, + gc_flag, qemu_flag, qemu_firmware, log, @@ -304,6 +334,7 @@ def test_suite( xip_flag, clean_up_flag, verbose_flag, + gc_flag, qemu_flag, qemu_firmware, log, @@ -414,6 +445,13 @@ def main(): dest="verbose_flag", help="Close real time output while running cases, only show last words of failed ones", ) + parser.add_argument( + "--gc", + action="store_true", + default=False, + dest="gc_flag", + help="Running with GC feature", + ) parser.add_argument( "cases", metavar="path_to__case", @@ -446,6 +484,7 @@ def main(): options.xip_flag, options.clean_up_flag, options.verbose_flag, + options.gc_flag, options.parl_flag, options.qemu_flag, options.qemu_firmware, @@ -469,6 +508,7 @@ def main(): options.xip_flag, options.clean_up_flag, options.verbose_flag, + options.gc_flag, options.qemu_flag, options.qemu_firmware, options.log diff --git a/tests/wamr-test-suites/spec-test-script/collect_coverage.sh b/tests/wamr-test-suites/spec-test-script/collect_coverage.sh new file mode 100755 index 00000000..316c30de --- /dev/null +++ b/tests/wamr-test-suites/spec-test-script/collect_coverage.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +readonly WORK_DIR=$PWD +readonly WAMR_DIR=${WORK_DIR}/../../.. +readonly DST_COV_FILE=$1 +readonly SRC_COV_DIR=$2 +readonly SRC_TEMP_COV_FILE=wamr_temp.lcov +readonly SRC_COV_FILE=wamr.lcov + +# get dest folder +dir=$(dirname ${DST_COV_FILE}) +pushd ${dir} > /dev/null 2>&1 +readonly DST_COV_DIR=${PWD} +popd > /dev/null 2>&1 + +if [[ ! -d ${SRC_COV_DIR} ]]; then + echo "${SRC_COV_DIR} doesn't exist, ignore code coverage collection" + exit +fi + +echo "Start to collect code coverage of ${SRC_COV_DIR} .." + +pushd ${SRC_COV_DIR} > /dev/null 2>&1 + +# collect all code coverage data +lcov -o ${SRC_TEMP_COV_FILE} -c -d . --rc lcov_branch_coverage=1 +# extract code coverage data of WAMR source files +lcov -r ${SRC_TEMP_COV_FILE} -o ${SRC_TEMP_COV_FILE} \ + -rc lcov_branch_coverage=1 \ + "*/usr/*" "*/_deps/*" "*/deps/*" "*/tests/unit/*" \ + "*/llvm/include/*" "*/include/llvm/*" "*/samples/*" \ + "*/app-framework/*" "*/test-tools/*" + +if [[ -s ${SRC_TEMP_COV_FILE} ]]; then + if [[ -s ${DST_COV_FILE} ]]; then + # merge code coverage data + lcov --rc lcov_branch_coverage=1 \ + --add-tracefile ${SRC_TEMP_COV_FILE} \ + -a ${DST_COV_FILE} -o ${SRC_COV_FILE} + # backup the original lcov file + cp -a ${DST_COV_FILE} "${DST_COV_FILE}.orig" + # replace the lcov file + cp -a ${SRC_COV_FILE} ${DST_COV_FILE} + else + cp -a ${SRC_TEMP_COV_FILE} ${SRC_COV_FILE} + cp -a ${SRC_COV_FILE} ${DST_COV_FILE} + fi + + # get ignored prefix path + dir=$(dirname ${WAMR_DIR}/../..) + pushd ${dir} > /dev/null 2>&1 + prefix_full_path=${PWD} + popd > /dev/null 2>&1 + + # generate html output for merged code coverage data + rm -fr ${DST_COV_DIR}/wamr-lcov + genhtml -t "WAMR Code Coverage" \ + --rc lcov_branch_coverage=1 --prefix=${prefix_full_path} \ + -o ${DST_COV_DIR}/wamr-lcov \ + ${DST_COV_FILE} + + cd ${DST_COV_DIR} + rm -f wamr-lcov.zip + zip -r -q -o wamr-lcov.zip wamr-lcov + rm -fr wamr-lcov + + echo "Code coverage file ${DST_COV_FILE} was generated or appended" + echo "Code coverage html ${DST_COV_DIR}/wamr-lcov.zip was generated" +else + echo "generate code coverage html failed" +fi + +echo "" + +popd > /dev/null 2>&1 diff --git a/tests/wamr-test-suites/spec-test-script/runtest.py b/tests/wamr-test-suites/spec-test-script/runtest.py index ef887c79..a1e505bd 100755 --- a/tests/wamr-test-suites/spec-test-script/runtest.py +++ b/tests/wamr-test-suites/spec-test-script/runtest.py @@ -38,7 +38,7 @@ log_file = None temp_file_repo = [] # to save the mapping of module files in /tmp by name -temp_module_table = {} +temp_module_table = {} def debug(data): if debug_file: @@ -230,6 +230,9 @@ parser.add_argument('--multi-module', default=False, action='store_true', parser.add_argument('--multi-thread', default=False, action='store_true', help="Enable Multi-thread") +parser.add_argument('--gc', default=False, action='store_true', + help='Test with GC') + parser.add_argument('--qemu', default=False, action='store_true', help="Enable QEMU") @@ -420,11 +423,20 @@ def parse_simple_const_w_type(number, type): number = float.fromhex(number) if '0x' in number else float(number) return number, "{:.7g}:{}".format(number, type) elif type == "ref.null": - # hard coding - return "extern", "extern:ref.null" + if number == "func": + return "func", "func:ref.null" + elif number == "extern": + return "extern", "extern:ref.null" + elif number == "any": + return "any", "any:ref.null" + else: + raise Exception("invalid value {} and type {}".format(number, type)) elif type == "ref.extern": number = int(number, 16) if '0x' in number else int(number) return number, "0x{:x}:ref.extern".format(number) + elif type == "ref.host": + number = int(number, 16) if '0x' in number else int(number) + return number, "0x{:x}:ref.host".format(number) else: raise Exception("invalid value {} and type {}".format(number, type)) @@ -440,6 +452,10 @@ def parse_assertion_value(val): type.const val ref.extern val ref.null ref_type + ref.array + ref.struct + ref.func + ref.i31 """ if not val: return None, "" @@ -453,6 +469,8 @@ def parse_assertion_value(val): if type in ["i32", "i64", "f32", "f64"]: return parse_simple_const_w_type(numbers[0], type) elif type == "ref": + if splitted[0] in ["ref.array", "ref.struct", "ref.func", "ref.i31"]: + return splitted[0] # need to distinguish between "ref.null" and "ref.extern" return parse_simple_const_w_type(numbers[0], splitted[0]) else: @@ -615,6 +633,9 @@ def simple_value_comparison(out, expected): elif "ref.extern" == expected_type: out_val_binary = out_val expected_val_binary = expected_val + elif "ref.host" == expected_type: + out_val_binary = out_val + expected_val_binary = expected_val else: assert(0), "unknown 'expected_type' {}".format(expected_type) @@ -637,8 +658,10 @@ def value_comparison(out, expected): if not expected: return False - assert(':' in out), "out should be in a form likes numbers:type, but {}".format(out) - assert(':' in expected), "expected should be in a form likes numbers:type, but {}".format(expected) + if not out in ["ref.array", "ref.struct", "ref.func", "ref.any", "ref.i31"]: + assert(':' in out), "out should be in a form likes numbers:type, but {}".format(out) + if not expected in ["ref.array", "ref.struct", "ref.func", "ref.any", "ref.i31"]: + assert(':' in expected), "expected should be in a form likes numbers:type, but {}".format(expected) if 'v128' in out: return vector_value_comparison(out, expected) @@ -761,6 +784,9 @@ def test_assert_return(r, opts, form): elif "ref.extern" == splitted[0]: number, _ = parse_simple_const_w_type(splitted[1], splitted[0]) args.append(str(number)) + elif "ref.host" == splitted[0]: + number, _ = parse_simple_const_w_type(splitted[1], splitted[0]) + args.append(str(number)) else: assert(0), "an unkonwn parameter type" @@ -769,7 +795,15 @@ def test_assert_return(r, opts, form): else: returns = re.split("\)\s*\(", m.group(3)[1:-1]) # processed numbers in strings - expected = [parse_assertion_value(v)[1] for v in returns] + if len(returns) == 1 and returns[0] in ["ref.array", "ref.struct", "ref.i31", + "ref.eq", "ref.any", "ref.extern", + "ref.func", "ref.null"]: + expected = [returns[0]] + elif len(returns) == 1 and returns[0] in ["func:ref.null", "any:ref.null", + "extern:ref.null"]: + expected = [returns[0]] + else: + expected = [parse_assertion_value(v)[1] for v in returns] expected = ",".join(expected) test_assert(r, opts, "return", "%s %s" % (func, " ".join(args)), expected) @@ -800,10 +834,10 @@ def test_assert_return(r, opts, form): if n.group(3) == '': args=[] else: - args = [re.split(' +', v)[1] for v in re.split("\)\s*\(", n.group(3)[1:-1])] - - # a workaround for "ref.null extern" and "ref.null func" - args = [ arg.replace('extern', 'null').replace('func', 'null') for arg in args] + # convert (ref.null extern/func) into (ref.null null) + n1 = n.group(3).replace("(ref.null extern)", "(ref.null null)") + n1 = n1.replace("ref.null func)", "(ref.null null)") + args = [re.split(' +', v)[1] for v in re.split("\)\s*\(", n1[1:-1])] _, expected = parse_assertion_value(n.group(4)[1:-1]) test_assert(r, opts, "return", "%s %s" % (func, " ".join(args)), expected) @@ -828,10 +862,10 @@ def test_assert_trap(r, opts, form): if m.group(2) == '': args = [] else: - args = [re.split(' +', v)[1] for v in re.split("\)\s*\(", m.group(2)[1:-1])] - - # workaround for "ref.null extern" - args = [ arg.replace('extern', 'null').replace('func', 'null') for arg in args] + # convert (ref.null extern/func) into (ref.null null) + m1 = m.group(2).replace("(ref.null extern)", "(ref.null null)") + m1 = m1.replace("ref.null func)", "(ref.null null)") + args = [re.split(' +', v)[1] for v in re.split("\)\s*\(", m1[1:-1])] expected = "Exception: %s" % m.group(3) test_assert(r, opts, "trap", "%s %s" % (func, " ".join(args)), expected) @@ -918,10 +952,11 @@ def compile_wast_to_wasm(form, wast_tempfile, wasm_tempfile, opts): log("Compiling WASM to '%s'" % wasm_tempfile) # default arguments - cmd = [opts.wast2wasm, - "--enable-thread", - "--no-check", - wast_tempfile, "-o", wasm_tempfile ] + if opts.gc: + cmd = [opts.wast2wasm, "-u", "-d", wast_tempfile, "-o", wasm_tempfile] + else: + cmd = [opts.wast2wasm, "--enable-thread", "--no-check", + wast_tempfile, "-o", wasm_tempfile ] # remove reference-type and bulk-memory enabling options since a WABT # commit 30c1e983d30b33a8004b39fd60cbd64477a7956c @@ -1023,18 +1058,18 @@ def run_wasm_with_repl(wasm_tempfile, aot_tempfile, opts, r): if (r != None): r.cleanup() r = Runner(cmd, no_pty=opts.no_pty) - + if opts.qemu: r.read_to_prompt(['nsh> '], 10) r.writeline("mount -t hostfs -o fs={} /tmp".format(tempfile.gettempdir())) r.read_to_prompt(['nsh> '], 10) r.writeline(" ".join(cmd_iwasm)) - + return r def create_tmpfiles(wast_name): tempfiles = [] - + (t1fd, wast_tempfile) = tempfile.mkstemp(suffix=".wast") (t2fd, wasm_tempfile) = tempfile.mkstemp(suffix=".wasm") tempfiles.append(wast_tempfile) diff --git a/tests/wamr-test-suites/test_wamr.sh b/tests/wamr-test-suites/test_wamr.sh index c220e6b3..c67a36ec 100755 --- a/tests/wamr-test-suites/test_wamr.sh +++ b/tests/wamr-test-suites/test_wamr.sh @@ -14,19 +14,23 @@ function help() { echo "test_wamr.sh [options]" echo "-c clean previous test results, not start test" - echo "-s {suite_name} test only one suite (spec)" - echo "-m set compile target of iwasm(x86_64\x86_32\armv7_vfp\thumbv7_vfp\riscv64_lp64d\riscv64_lp64)" - echo "-t set compile type of iwasm(classic-interp\fast-interp\jit\aot\fast-jit\multi-tier-jit)" + echo "-s {suite_name} test only one suite (spec|wasi_certification)" + echo "-m set compile target of iwasm(x86_64|x86_32|armv7_vfp|thumbv7_vfp|riscv64_lp64d|riscv64_lp64)" + echo "-t set compile type of iwasm(classic-interp|fast-interp|jit|aot|fast-jit|multi-tier-jit)" echo "-M enable multi module feature" echo "-p enable multi thread feature" echo "-S enable SIMD feature" + echo "-G enable GC feature" echo "-X enable XIP feature" echo "-x test SGX" + echo "-w enable WASI threads" echo "-b use the wabt binary release package instead of compiling from the source code" + echo "-g build iwasm with debug version" + echo "-v enable GC heap verification" echo "-P run the spec test parallelly" echo "-Q enable qemu" echo "-F set the firmware path used by qemu" - echo "-w enable WASI threads" + echo "-C enable code coverage collect" } OPT_PARSED="" @@ -40,7 +44,10 @@ ENABLE_MULTI_MODULE=0 ENABLE_MULTI_THREAD=0 COLLECT_CODE_COVERAGE=0 ENABLE_SIMD=0 +ENABLE_GC=0 ENABLE_XIP=0 +ENABLE_DEBUG_VERSION=0 +ENABLE_GC_HEAP_VERIFY=0 #unit test case arrary TEST_CASE_ARR=() SGX_OPT="" @@ -50,7 +57,7 @@ ENABLE_QEMU=0 QEMU_FIRMWARE="" WASI_TESTSUITE_COMMIT="b18247e2161bea263fe924b8189c67b1d2d10a10" -while getopts ":s:cabt:m:wMCpSXxPQF:" opt +while getopts ":s:cabgvt:m:MCpSXxwPGQF:" opt do OPT_PARSED="TRUE" case $opt in @@ -72,8 +79,9 @@ do c) read -t 5 -p "Are you sure to delete all reports. y/n " cmd if [[ $cmd == "y" && $(ls -A workspace/report) ]];then - rm -r workspace/report/* - echo "cleaned all reports" + rm -fr workspace/report/* + rm -fr /tmp/*.wasm /tmp/*.wast /tmp/*.aot + echo "cleaned all reports and temp files" fi exit 0;; a) @@ -128,6 +136,18 @@ do echo "test SGX" SGX_OPT="--sgx" ;; + g) + echo "enable build iwasm with debug version" + ENABLE_DEBUG_VERSION=1 + ;; + v) + echo "enable GC heap verification" + ENABLE_GC_HEAP_VERIFY=1 + ;; + G) + echo "enable GC feature" + ENABLE_GC=1 + ;; P) PARALLELISM=1 ;; @@ -164,7 +184,6 @@ readonly DATE=$(date +%Y-%m-%d_%H:%M:%S) readonly REPORT_DIR=${WORK_DIR}/report/${DATE} mkdir -p ${REPORT_DIR} -# TODO: a strong assumation about a link to the WAMR project readonly WAMR_DIR=${WORK_DIR}/../../.. if [[ ${SGX_OPT} == "--sgx" ]];then @@ -198,14 +217,16 @@ readonly ORC_EAGER_JIT_COMPILE_FLAGS="\ -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_INTERP=0 \ -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_AOT=1 \ -DWAMR_BUILD_LAZY_JIT=0 \ - -DWAMR_BUILD_SPEC_TEST=1" + -DWAMR_BUILD_SPEC_TEST=1 \ + -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}" readonly ORC_LAZY_JIT_COMPILE_FLAGS="\ -DWAMR_BUILD_TARGET=${TARGET} \ -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_INTERP=0 \ -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_AOT=1 \ -DWAMR_BUILD_LAZY_JIT=1 \ - -DWAMR_BUILD_SPEC_TEST=1" + -DWAMR_BUILD_SPEC_TEST=1 \ + -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}" readonly AOT_COMPILE_FLAGS="\ -DWAMR_BUILD_TARGET=${TARGET} \ @@ -219,13 +240,15 @@ readonly FAST_JIT_COMPILE_FLAGS="\ -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_INTERP=0 \ -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_AOT=0 \ -DWAMR_BUILD_FAST_JIT=1 \ - -DWAMR_BUILD_SPEC_TEST=1" + -DWAMR_BUILD_SPEC_TEST=1 \ + -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}" readonly MULTI_TIER_JIT_COMPILE_FLAGS="\ -DWAMR_BUILD_TARGET=${TARGET} \ -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_INTERP=0 \ -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_JIT=1 \ - -DWAMR_BUILD_SPEC_TEST=1" + -DWAMR_BUILD_SPEC_TEST=1 \ + -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}" readonly COMPILE_FLAGS=( "${CLASSIC_INTERP_COMPILE_FLAGS}" @@ -237,39 +260,19 @@ readonly COMPILE_FLAGS=( "${MULTI_TIER_JIT_COMPILE_FLAGS}" ) -# TODO: with libiwasm.so only function unit_test() { echo "Now start unit tests" cd ${WORK_DIR} - readonly UNIT_CASES="wasm-vm host-tool utils" + rm -fr unittest-build && mkdir unittest-build + cd unittest-build echo "Build unit test" touch ${REPORT_DIR}/unit_test_report.txt - - for compile_flag in "${COMPILE_FLAGS[@]}"; do - echo "Build unit test with compile flags with " ${compile_flag} - - # keep going and do not care if it is success or not - make -ki clean | true - cmake ${compile_flag} ${WORK_DIR}/../../unit && make -j 4 - if [ "$?" != 0 ];then - echo -e "build unit test failed, you may need to change wamr into dev/aot branch and ensure llvm is built" - exit 1 - fi - - echo ${compile_flag} >> ${REPORT_DIR}/unit_test_report.txt - - for case in ${UNIT_CASES} - do - echo "run ${case} ..." - cd ./${case}/ - ./${case/-/_}"_test" | tee -a ${REPORT_DIR}/unit_test_report.txt - cd - - echo "finish ${case}" - done - done + cmake ${WORK_DIR}/../../unit -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE} + make -j + make test | tee -a ${REPORT_DIR}/unit_test_report.txt echo "Finish unit tests" } @@ -347,6 +350,27 @@ function spec_test() git apply ../../spec-test-script/thread_proposal_fix_atomic_case.patch fi + # update GC cases + if [[ ${ENABLE_GC} == 1 ]]; then + echo "checkout spec for GC proposal" + + popd + rm -fr spec + # check spec test cases for GC + git clone -b main --single-branch https://github.com/WebAssembly/gc.git spec + pushd spec + + git restore . && git clean -ffd . + # Sync constant expression descriptions + git reset --hard 62beb94ddd41987517781732f17f213d8b866dcc + git apply ../../spec-test-script/gc_ignore_cases.patch + + echo "compile the reference intepreter" + pushd interpreter + make opt + popd + fi + popd echo $(pwd) @@ -446,6 +470,10 @@ function spec_test() ARGS_FOR_SPEC_TEST+="--parl " fi + if [[ ${ENABLE_GC} == 1 ]]; then + ARGS_FOR_SPEC_TEST+="--gc " + fi + if [[ ${ENABLE_QEMU} == 1 ]]; then ARGS_FOR_SPEC_TEST+="--qemu " ARGS_FOR_SPEC_TEST+="--qemu-firmware ${QEMU_FIRMWARE} " @@ -484,7 +512,7 @@ function wasi_test() function wasi_certification_test() { - echo "Now start wasi tests" + echo "Now start wasi certification tests" cd ${WORK_DIR} if [ ! -d "wasi-testsuite" ]; then @@ -514,7 +542,6 @@ function polybench_test() if [[ $1 == "aot" || $1 == "jit" ]];then ./build.sh AOT ${SGX_OPT} ./test_aot.sh $1 ${SGX_OPT} - else ./build.sh ./test_interp.sh ${SGX_OPT} @@ -524,6 +551,22 @@ function polybench_test() echo "Finish polybench tests" } +function libsodium_test() +{ + echo "Now start libsodium tests" + + cd ${WORK_DIR}/../libsodium + if [[ $1 == "aot" || $1 == "jit" ]];then + ./build.sh ${SGX_OPT} + ./test_aot.sh $1 ${SGX_OPT} + else + ./test_interp.sh ${SGX_OPT} + fi + cp report.txt ${REPORT_DIR}/libsodium_$1_test_report.txt + + echo "Finish libsodium tests" +} + function malformed_test() { # build iwasm firstly @@ -535,14 +578,13 @@ function standalone_test() { cd ${WORK_DIR}/../../standalone - args="" + args="--$1" - [[ $1 == "aot" ]] && args="$args --aot" || args="$args --no-aot" [[ ${SGX_OPT} == "--sgx" ]] && args="$args --sgx" || args="$args --no-sgx" - if [[ ${ENABLE_MULTI_THREAD} == 1 ]];then - args="$args --thread" - fi + [[ ${ENABLE_MULTI_THREAD} == 1 ]] && args="$args --thread" && args="$args --no-thread" + + [[ ${ENABLE_SIMD} == 1 ]] && args="$args --simd" && args="$args --no-simd" ./standalone.sh $args | tee ${REPORT_DIR}/standalone_$1_test_report.txt } @@ -589,7 +631,7 @@ function build_wamrc() && ./build_llvm.sh \ && if [ -d build ]; then rm -r build/*; else mkdir build; fi \ && cd build \ - && cmake .. \ + && cmake .. -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE} \ && make -j 4 } @@ -602,15 +644,33 @@ function build_wamrc() function collect_coverage() { - if [[ ${COLLECT_CODE_COVERAGE} == 1 ]];then - cd ${IWASM_LINUX_ROOT_DIR}/build - lcov -t "iwasm code coverage" -o iwasm.info -c -d . - genhtml -o iwasm-gcov iwasm.info - [[ -d iwasm-gcov ]] && \ - cp -r iwasm-gcov ${REPORT_DIR}/$1_iwasm_gcov || \ - echo "generate code coverage html failed" + if [[ ${COLLECT_CODE_COVERAGE} == 1 ]]; then + ln -sf ${WORK_DIR}/../spec-test-script/collect_coverage.sh ${WORK_DIR} + + CODE_COV_FILE="" + if [[ -z "${COV_FILE}" ]]; then + CODE_COV_FILE="${WORK_DIR}/wamr.lcov" + else + CODE_COV_FILE="${COV_FILE}" + fi + + pushd ${WORK_DIR} > /dev/null 2>&1 + echo "Collect code coverage of iwasm" + ./collect_coverage.sh ${CODE_COV_FILE} ${IWASM_LINUX_ROOT_DIR}/build + if [[ $1 == "llvm-aot" ]]; then + echo "Collect code coverage of wamrc" + ./collect_coverage.sh ${CODE_COV_FILE} ${WAMR_DIR}/wamr-compiler/build + fi + for suite in "${TEST_CASE_ARR[@]}"; do + if [[ ${suite} = "unit" ]]; then + echo "Collect code coverage of unit test" + ./collect_coverage.sh ${CODE_COV_FILE} ${WORK_DIR}/unittest-build + break + fi + done + popd > /dev/null 2>&1 else - echo "will not collect code coverage" + echo "code coverage isn't collected" fi } @@ -639,10 +699,22 @@ function trigger() EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_SIMD=0" fi + if [[ ${ENABLE_GC} == 1 ]]; then + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_GC=1" + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_REF_TYPES=1" + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_BULK_MEMORY=1" + fi + + if [[ ${ENABLE_DEBUG_VERSION} == 1 ]]; then + EXTRA_COMPILE_FLAGS+=" -DCMAKE_BUILD_TYPE=Debug" + fi + + if [[ ${ENABLE_GC_HEAP_VERIFY} == 1 ]]; then + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_GC_HEAP_VERIFY=1" + fi + if [[ ${ENABLE_WASI_THREADS} == 1 ]]; then EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_LIB_WASI_THREADS=1" - else - EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_LIB_WASI_THREADS=0" fi for t in "${TYPE[@]}"; do @@ -692,18 +764,18 @@ function trigger() echo "work in orc jit eager compilation mode" BUILD_FLAGS="$ORC_EAGER_JIT_COMPILE_FLAGS $EXTRA_COMPILE_FLAGS" build_iwasm_with_cfg $BUILD_FLAGS - build_wamrc for suite in "${TEST_CASE_ARR[@]}"; do $suite"_test" jit done + collect_coverage llvm-jit echo "work in orc jit lazy compilation mode" BUILD_FLAGS="$ORC_EAGER_JIT_COMPILE_FLAGS $EXTRA_COMPILE_FLAGS" build_iwasm_with_cfg $BUILD_FLAGS - build_wamrc for suite in "${TEST_CASE_ARR[@]}"; do $suite"_test" jit done + collect_coverage llvm-jit ;; "aot") @@ -717,7 +789,7 @@ function trigger() for suite in "${TEST_CASE_ARR[@]}"; do $suite"_test" aot done - collect_coverage aot + collect_coverage llvm-aot ;; "fast-jit") @@ -728,6 +800,7 @@ function trigger() for suite in "${TEST_CASE_ARR[@]}"; do $suite"_test" fast-jit done + collect_coverage fast-jit ;; "multi-tier-jit") @@ -738,6 +811,7 @@ function trigger() for suite in "${TEST_CASE_ARR[@]}"; do $suite"_test" multi-tier-jit done + collect_coverage multi-tier-jit ;; *) @@ -748,11 +822,19 @@ function trigger() } # if collect code coverage, ignore -s, test all test cases. -if [[ $TEST_CASE_ARR && $COLLECT_CODE_COVERAGE != 1 ]];then +if [[ $TEST_CASE_ARR ]];then trigger || (echo "TEST FAILED"; exit 1) else - # test all suite, ignore polybench because of long time cost - TEST_CASE_ARR=("sightglass" "spec" "wasi" "malformed" "standalone") + # test all suite, ignore polybench and libsodium because of long time cost + TEST_CASE_ARR=("spec") + : ' + if [[ $COLLECT_CODE_COVERAGE == 1 ]];then + # add polybench if collecting code coverage data + TEST_CASE_ARR+=("polybench") + # add libsodium if needed, which takes long time to run + TEST_CASE_ARR+=("libsodium") + fi + ' trigger || (echo "TEST FAILED"; exit 1) # Add more suites here fi diff --git a/wamr-compiler/CMakeLists.txt b/wamr-compiler/CMakeLists.txt index 18d65c07..0ae821af 100644 --- a/wamr-compiler/CMakeLists.txt +++ b/wamr-compiler/CMakeLists.txt @@ -166,6 +166,12 @@ if (WAMR_BUILD_DEBUG_AOT EQUAL 1) message(STATUS "find lldb ${LLDB_ALL_PLUGINS} in: ${LLVM_LIBRARY_DIRS}") endif() +if ("$ENV{COLLECT_CODE_COVERAGE}" STREQUAL "1" OR COLLECT_CODE_COVERAGE EQUAL 1) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage") + message ("-- Collect code coverage enabled") +endif () + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) if(NOT MSVC) set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections")