From 5a7fbda7acb663364860fb347390fc3b2a39ac8e Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Tue, 7 Mar 2023 08:45:03 +0800 Subject: [PATCH] Enable to collect code coverage of samples/simple (#2003) --- build-scripts/config_common.cmake | 1 + core/app-mgr/app-manager/module_wasm_app.c | 8 ++ samples/simple/CMakeLists.txt | 5 + samples/simple/build.sh | 6 +- samples/simple/sample_test_run.py | 91 ++++++++++++++++--- .../spec-test-script/collect_coverage.sh | 12 ++- tests/wamr-test-suites/test_wamr.sh | 53 ++++++++++- wamr-sdk/build_sdk.sh | 6 +- 8 files changed, 154 insertions(+), 28 deletions(-) diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index 3ac60846..82301424 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -348,5 +348,6 @@ endif () if ("$ENV{COLLECT_CODE_COVERAGE}" STREQUAL "1" OR COLLECT_CODE_COVERAGE EQUAL 1) set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage") + add_definitions (-DCOLLECT_CODE_COVERAGE) message (" Collect code coverage enabled") endif () diff --git a/core/app-mgr/app-manager/module_wasm_app.c b/core/app-mgr/app-manager/module_wasm_app.c index 7b826d6d..2005ad8e 100644 --- a/core/app-mgr/app-manager/module_wasm_app.c +++ b/core/app-mgr/app-manager/module_wasm_app.c @@ -993,6 +993,14 @@ wasm_app_module_uninstall(request_t *msg) app_manager_printf("Uninstall WASM app successful!\n"); +#ifdef COLLECT_CODE_COVERAGE + /* Exit app manager so as to collect code coverage data */ + if (!strcmp(m_name, "__exit_app_manager__")) { + app_manager_printf("Exit app manager\n"); + bh_queue_exit_loop_run(get_app_manager_queue()); + } +#endif + #if VALGRIND_CHECK != 0 bh_queue_exit_loop_run(get_app_manager_queue()); #endif diff --git a/samples/simple/CMakeLists.txt b/samples/simple/CMakeLists.txt index 90d6c232..f3a0848f 100644 --- a/samples/simple/CMakeLists.txt +++ b/samples/simple/CMakeLists.txt @@ -12,6 +12,11 @@ message(STATUS "WAMR_BUILD_SDK_PROFILE=${WAMR_BUILD_SDK_PROFILE}") set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") +if ("$ENV{COLLECT_CODE_COVERAGE}" STREQUAL "1" OR COLLECT_CODE_COVERAGE EQUAL 1) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage") +endif () + set (WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) ## use library and headers in the SDK diff --git a/samples/simple/build.sh b/samples/simple/build.sh index aab9997e..4e815515 100755 --- a/samples/simple/build.sh +++ b/samples/simple/build.sh @@ -56,7 +56,7 @@ done if [ "$CLEAN" = "TRUE" ]; then - rm -rf $CURR_DIR/cmake_build + rm -rf $CURR_DIR/cmake-build fi @@ -107,8 +107,8 @@ cd ${WAMR_DIR}/wamr-sdk echo "#####################build simple project" cd ${CURR_DIR} -mkdir -p cmake_build/$PROFILE -cd cmake_build/$PROFILE +mkdir -p cmake-build/$PROFILE +cd cmake-build/$PROFILE cmake ../.. -DWAMR_BUILD_SDK_PROFILE=$PROFILE $CM_TOOLCHAIN $CM_BUILD_TYPE make if [ $? != 0 ];then diff --git a/samples/simple/sample_test_run.py b/samples/simple/sample_test_run.py index d9e007c0..ed4916c8 100755 --- a/samples/simple/sample_test_run.py +++ b/samples/simple/sample_test_run.py @@ -11,6 +11,7 @@ import sys import time import traceback +WAMRC_CMD = "../../wamr-compiler/build/wamrc" def start_server(cwd): """ @@ -80,8 +81,31 @@ def main(): """ parser = argparse.ArgumentParser(description="run the sample and examine outputs") parser.add_argument("working_directory", type=str) + parser.add_argument("--aot", action='store_true', help="Test with AOT") args = parser.parse_args() + test_aot = False + suffix = ".wasm" + if not args.aot: + print("Test with interpreter mode") + else: + print("Test with AOT mode") + test_aot = True + suffix = ".aot" + wasm_files = [ "timer", "sensor", "connection", + "event_publisher", "event_subscriber", + "request_handler", "request_sender" ] + work_dir = args.working_directory + wasm_apps_dir = work_dir + "/wasm-apps" + print("Compile wasm app into aot files") + for wasm_file in wasm_files: + CMD = [] + CMD.append(WAMRC_CMD) + CMD.append("-o") + CMD.append(wasm_apps_dir + "/" + wasm_file + ".aot") + CMD.append(wasm_apps_dir + "/" + wasm_file + ".wasm") + subprocess.check_call(CMD) + ret = 1 app_server = None try: @@ -90,35 +114,47 @@ def main(): # wait for a second time.sleep(1) - print("--> Install timer.wasm...") + print("--> Install timer" + suffix + "...") install_wasm_application( - "timer", "./wasm-apps/timer.wasm", args.working_directory + "timer", "./wasm-apps/timer" + suffix, args.working_directory ) - print("--> Install event_publisher.wasm...") + # wait for a second + time.sleep(3) + + print("--> Query all installed applications...") + query_installed_application(args.working_directory) + + print("--> Install event_publisher" + suffix + "...") install_wasm_application( "event_publisher", - "./wasm-apps/event_publisher.wasm", + "./wasm-apps/event_publisher" + suffix, args.working_directory, ) - print("--> Install event_subscriber.wasm...") + print("--> Install event_subscriber" + suffix + "...") install_wasm_application( "event_subscriber", - "./wasm-apps/event_subscriber.wasm", + "./wasm-apps/event_subscriber" + suffix, args.working_directory, ) - print("--> Uninstall timer.wasm...") + print("--> Query all installed applications...") + query_installed_application(args.working_directory) + + print("--> Uninstall timer" + suffix + "...") uninstall_wasm_application("timer", args.working_directory) - print("--> Uninstall event_publisher.wasm...") + print("--> Query all installed applications...") + query_installed_application(args.working_directory) + + print("--> Uninstall event_publisher" + suffix + "...") uninstall_wasm_application( "event_publisher", args.working_directory, ) - print("--> Uninstall event_subscriber.wasm...") + print("--> Uninstall event_subscriber" + suffix + "...") uninstall_wasm_application( "event_subscriber", args.working_directory, @@ -127,26 +163,55 @@ def main(): print("--> Query all installed applications...") query_installed_application(args.working_directory) - print("--> Install request_handler.wasm...") + print("--> Install request_handler" + suffix + "...") install_wasm_application( "request_handler", - "./wasm-apps/request_handler.wasm", + "./wasm-apps/request_handler" + suffix, args.working_directory, ) print("--> Query again...") query_installed_application(args.working_directory) - print("--> Install request_sender.wasm...") + print("--> Install request_sender" + suffix + "...") install_wasm_application( "request_sender", - "./wasm-apps/request_sender.wasm", + "./wasm-apps/request_sender" + suffix, args.working_directory, ) print("--> Send GET to the Wasm application named request_handler...") send_get_to_wasm_application("request_handler", "/url1", args.working_directory) + print("--> Uninstall request_handler" + suffix + "...") + uninstall_wasm_application( + "request_handler", + args.working_directory, + ) + + print("--> Uninstall request_sender" + suffix + "...") + uninstall_wasm_application( + "request_sender", + args.working_directory, + ) + + # Install a wasm app named "__exit_app_manager__" just to make app manager exit + # while the wasm app is uninstalled, so as to collect the code coverage data. + # Only available when collecting code coverage is enabled. + print("--> Install timer" + suffix + "...") + install_wasm_application( + "__exit_app_manager__", "./wasm-apps/timer" + suffix, args.working_directory + ) + + print("--> Uninstall timer" + suffix + "...") + uninstall_wasm_application( + "__exit_app_manager__", + args.working_directory, + ) + + # wait for a second + time.sleep(1) + print("--> All pass") ret = 0 except AssertionError: diff --git a/tests/wamr-test-suites/spec-test-script/collect_coverage.sh b/tests/wamr-test-suites/spec-test-script/collect_coverage.sh index 316c30de..09b1f465 100755 --- a/tests/wamr-test-suites/spec-test-script/collect_coverage.sh +++ b/tests/wamr-test-suites/spec-test-script/collect_coverage.sh @@ -28,13 +28,14 @@ echo "Start to collect code coverage of ${SRC_COV_DIR} .." pushd ${SRC_COV_DIR} > /dev/null 2>&1 # collect all code coverage data -lcov -o ${SRC_TEMP_COV_FILE} -c -d . --rc lcov_branch_coverage=1 +lcov -q -o ${SRC_TEMP_COV_FILE} -c -d . --rc lcov_branch_coverage=1 # extract code coverage data of WAMR source files -lcov -r ${SRC_TEMP_COV_FILE} -o ${SRC_TEMP_COV_FILE} \ +lcov -q -r ${SRC_TEMP_COV_FILE} -o ${SRC_TEMP_COV_FILE} \ -rc lcov_branch_coverage=1 \ "*/usr/*" "*/_deps/*" "*/deps/*" "*/tests/unit/*" \ "*/llvm/include/*" "*/include/llvm/*" "*/samples/*" \ - "*/app-framework/*" "*/test-tools/*" + "*/app-framework/*" "*/app-mgr/*" "*/test-tools/*" \ + "*/tests/standalone/*" "*/tests/*" if [[ -s ${SRC_TEMP_COV_FILE} ]]; then if [[ -s ${DST_COV_FILE} ]]; then @@ -46,9 +47,11 @@ if [[ -s ${SRC_TEMP_COV_FILE} ]]; then cp -a ${DST_COV_FILE} "${DST_COV_FILE}.orig" # replace the lcov file cp -a ${SRC_COV_FILE} ${DST_COV_FILE} + echo "Code coverage file ${DST_COV_FILE} was appended" else cp -a ${SRC_TEMP_COV_FILE} ${SRC_COV_FILE} cp -a ${SRC_COV_FILE} ${DST_COV_FILE} + echo "Code coverage file ${DST_COV_FILE} was generated" fi # get ignored prefix path @@ -59,7 +62,7 @@ if [[ -s ${SRC_TEMP_COV_FILE} ]]; then # generate html output for merged code coverage data rm -fr ${DST_COV_DIR}/wamr-lcov - genhtml -t "WAMR Code Coverage" \ + genhtml -q -t "WAMR Code Coverage" \ --rc lcov_branch_coverage=1 --prefix=${prefix_full_path} \ -o ${DST_COV_DIR}/wamr-lcov \ ${DST_COV_FILE} @@ -69,7 +72,6 @@ if [[ -s ${SRC_TEMP_COV_FILE} ]]; then zip -r -q -o wamr-lcov.zip wamr-lcov rm -fr wamr-lcov - echo "Code coverage file ${DST_COV_FILE} was generated or appended" echo "Code coverage html ${DST_COV_DIR}/wamr-lcov.zip was generated" else echo "generate code coverage html failed" diff --git a/tests/wamr-test-suites/test_wamr.sh b/tests/wamr-test-suites/test_wamr.sh index 4fa730e7..cf0bc970 100755 --- a/tests/wamr-test-suites/test_wamr.sh +++ b/tests/wamr-test-suites/test_wamr.sh @@ -574,19 +574,64 @@ function malformed_test() ./malformed_test.py --run ${IWASM_CMD} | tee ${REPORT_DIR}/malfomed_$1_test_report.txt } +function collect_standalone() +{ + if [[ ${COLLECT_CODE_COVERAGE} == 1 ]]; then + pushd ${WORK_DIR} > /dev/null 2>&1 + + CODE_COV_FILE="" + if [[ -z "${CODE_COV_FILE}" ]]; then + CODE_COV_FILE="${WORK_DIR}/wamr.lcov" + else + CODE_COV_FILE="${CODE_COV_FILE}" + fi + + STANDALONE_DIR=${WORK_DIR}/../../standalone + + echo "Collect code coverage of standalone dump-call-stack" + ./collect_coverage.sh "${CODE_COV_FILE}" "${STANDALONE_DIR}/dump-call-stack/build" + echo "Collect code coverage of standalone dump-mem-profiling" + ./collect_coverage.sh "${CODE_COV_FILE}" "${STANDALONE_DIR}/dump-mem-profiling/build" + echo "Collect code coverage of standalone dump-perf-profiling" + ./collect_coverage.sh "${CODE_COV_FILE}" "${STANDALONE_DIR}/dump-perf-profiling/build" + if [[ $1 == "aot" ]]; then + echo "Collect code coverage of standalone pad-test" + ./collect_coverage.sh "${CODE_COV_FILE}" "${STANDALONE_DIR}/pad-test/build" + fi + echo "Collect code coverage of standalone test-invoke-native" + ./collect_coverage.sh "${CODE_COV_FILE}" "${STANDALONE_DIR}/test-invoke-native/build" + echo "Collect code coverage of standalone test-running-modes" + ./collect_coverage.sh "${CODE_COV_FILE}" "${STANDALONE_DIR}/test-running-modes/build" + echo "Collect code coverage of standalone test-running-modes/c-embed" + ./collect_coverage.sh "${CODE_COV_FILE}" "${STANDALONE_DIR}/test-running-modes/c-embed/build" + echo "Collect code coverage of standalone test-ts2" + ./collect_coverage.sh "${CODE_COV_FILE}" "${STANDALONE_DIR}/test-ts2/build" + + popd > /dev/null 2>&1 + fi +} + function standalone_test() { + if [[ ${COLLECT_CODE_COVERAGE} == 1 ]]; then + export COLLECT_CODE_COVERAGE=1 + fi + cd ${WORK_DIR}/../../standalone args="--$1" [[ ${SGX_OPT} == "--sgx" ]] && args="$args --sgx" || args="$args --no-sgx" - [[ ${ENABLE_MULTI_THREAD} == 1 ]] && args="$args --thread" && args="$args --no-thread" + [[ ${ENABLE_MULTI_THREAD} == 1 ]] && args="$args --thread" || args="$args --no-thread" - [[ ${ENABLE_SIMD} == 1 ]] && args="$args --simd" && args="$args --no-simd" + [[ ${ENABLE_SIMD} == 1 ]] && args="$args --simd" || args="$args --no-simd" + + args="$args ${TARGET}" ./standalone.sh $args | tee ${REPORT_DIR}/standalone_$1_test_report.txt + + collect_standalone "$1" } function build_iwasm_with_cfg() @@ -648,10 +693,10 @@ function collect_coverage() ln -sf ${WORK_DIR}/../spec-test-script/collect_coverage.sh ${WORK_DIR} CODE_COV_FILE="" - if [[ -z "${COV_FILE}" ]]; then + if [[ -z "${CODE_COV_FILE}" ]]; then CODE_COV_FILE="${WORK_DIR}/wamr.lcov" else - CODE_COV_FILE="${COV_FILE}" + CODE_COV_FILE="${CODE_COV_FILE}" fi pushd ${WORK_DIR} > /dev/null 2>&1 diff --git a/wamr-sdk/build_sdk.sh b/wamr-sdk/build_sdk.sh index 53856e9c..954584f6 100755 --- a/wamr-sdk/build_sdk.sh +++ b/wamr-sdk/build_sdk.sh @@ -224,8 +224,8 @@ echo -e "\n\n" echo "############## Start to build runtime sdk ###############" cd ${sdk_root}/runtime -rm -fr build_runtime_sdk && mkdir build_runtime_sdk -cd build_runtime_sdk +rm -fr build-runtime-sdk && mkdir build-runtime-sdk +cd build-runtime-sdk cmake .. $CM_DEXTRA_SDK_INCLUDE_PATH \ -DWAMR_BUILD_SDK_PROFILE=${PROFILE} \ -DCONFIG_PATH=${wamr_config_cmake_file} \ @@ -249,6 +249,6 @@ fi cd .. -rm -fr build_runtime_sdk +rm -fr build-runtime-sdk exit 0