CloudEdge/wasm-micro-runtime
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

wamr-test-suites: Add support for ARM/RISCV by QEMU (#1704)

Browse Source 
Enhance wamr-test-suites to add QEMU and Firmware options:
`./test_wamr.sh -Q <qemu> -F <firmware>`
This commit is contained in:
Huang Qi 2022-11-28 10:45:32 +08:00 committed by GitHub
parent 93d3d09aa1
commit 9c5e1cb600
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 212 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
product-mini/platforms/posix/main.c
View File

@ -6,6 +6,7 @@
#ifndef _GNU_SOURCE #ifndef _GNU_SOURCE
#define _GNU_SOURCE #define _GNU_SOURCE
#endif #endif
#include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
@ -149,7 +150,9 @@ app_instance_repl(wasm_module_inst_t module_inst)
size_t len = 0; size_t len = 0;
ssize_t n; ssize_t n;
while ((printf("webassembly> "), n = getline(&cmd, &len, stdin)) != -1) { while ((printf("webassembly> "), fflush(stdout),
n = getline(&cmd, &len, stdin))
!= -1) {
bh_assert(n > 0); bh_assert(n > 0);
if (cmd[n - 1] == '\n') { if (cmd[n - 1] == '\n') {
if (n == 1) if (n == 1)

66
tests/wamr-test-suites/spec-test-script/all.py
View File

@ -5,14 +5,9 @@
# #
import argparse import argparse
import hashlib
import multiprocessing as mp import multiprocessing as mp
import os import os
import pathlib import pathlib
import random
import shlex
import shutil
import string
import subprocess import subprocess
import sys import sys
import time import time
@ -24,6 +19,7 @@ The script itself has to be put under the same directory with the "spec".
PLATFORM_NAME = os.uname().sysname.lower() PLATFORM_NAME = os.uname().sysname.lower()
IWASM_CMD = "../../../product-mini/platforms/" + PLATFORM_NAME + "/build/iwasm" IWASM_CMD = "../../../product-mini/platforms/" + PLATFORM_NAME + "/build/iwasm"
IWASM_SGX_CMD = "../../../product-mini/platforms/linux-sgx/enclave-sample/iwasm" IWASM_SGX_CMD = "../../../product-mini/platforms/linux-sgx/enclave-sample/iwasm"
IWASM_QEMU_CMD = "iwasm"
SPEC_TEST_DIR = "spec/test/core" SPEC_TEST_DIR = "spec/test/core"
WAST2WASM_CMD = "./wabt/out/gcc/Release/wat2wasm" WAST2WASM_CMD = "./wabt/out/gcc/Release/wat2wasm"
WAMRC_CMD = "../../../wamr-compiler/build/wamrc" WAMRC_CMD = "../../../wamr-compiler/build/wamrc"
@ -32,8 +28,11 @@ WAMRC_CMD = "../../../wamr-compiler/build/wamrc"
class TargetAction(argparse.Action): class TargetAction(argparse.Action):
TARGET_MAP = { TARGET_MAP = {
"ARMV7_VFP": "armv7", "ARMV7_VFP": "armv7",
"RISCV64": "riscv64_lp64d", "RISCV32": "riscv32_ilp32",
"RISCV64_LP64": "riscv64_lp64d", "RISCV32_ILP32": "riscv32_ilp32",
"RISCV32_ILP32D": "riscv32_ilp32d",
"RISCV64": "riscv64_lp64",
"RISCV64_LP64": "riscv64_lp64",
"RISCV64_LP64D": "riscv64_lp64", "RISCV64_LP64D": "riscv64_lp64",
"THUMBV7_VFP": "thumbv7", "THUMBV7_VFP": "thumbv7",
"X86_32": "i386", "X86_32": "i386",
@ -105,6 +104,8 @@ def test_case(
xip_flag=False, xip_flag=False,
clean_up_flag=True, clean_up_flag=True,
verbose_flag=True, verbose_flag=True,
qemu_flag=False,
qemu_firmware='',
): ):
case_path = pathlib.Path(case_path).resolve() case_path = pathlib.Path(case_path).resolve()
case_name = case_path.stem case_name = case_path.stem
@ -125,14 +126,23 @@ def test_case(
CMD.append("--wast2wasm") CMD.append("--wast2wasm")
CMD.append(WAST2WASM_CMD) CMD.append(WAST2WASM_CMD)
CMD.append("--interpreter") CMD.append("--interpreter")
CMD.append(IWASM_CMD if not sgx_flag else IWASM_SGX_CMD) if sgx_flag:
CMD.append(IWASM_SGX_CMD)
elif qemu_flag:
CMD.append(IWASM_QEMU_CMD)
else:
CMD.append(IWASM_CMD)
CMD.append("--aot-compiler") CMD.append("--aot-compiler")
CMD.append(WAMRC_CMD) CMD.append(WAMRC_CMD)
if aot_flag: if aot_flag:
CMD.append("--aot") CMD.append("--aot")
CMD.append("--aot-target")
CMD.append(target) CMD.append("--target")
CMD.append(target)
if multi_module_flag:
CMD.append("--multi-module")
if multi_thread_flag: if multi_thread_flag:
CMD.append("--multi-thread") CMD.append("--multi-thread")
@ -146,6 +156,11 @@ def test_case(
if xip_flag: if xip_flag:
CMD.append("--xip") CMD.append("--xip")
if qemu_flag:
CMD.append("--qemu")
CMD.append("--qemu-firmware")
CMD.append(qemu_firmware)
if not clean_up_flag: if not clean_up_flag:
CMD.append("--no_cleanup") CMD.append("--no_cleanup")
@ -206,6 +221,8 @@ def test_suite(
clean_up_flag=True, clean_up_flag=True,
verbose_flag=True, verbose_flag=True,
parl_flag=False, parl_flag=False,
qemu_flag=False,
qemu_firmware=''
): ):
suite_path = pathlib.Path(SPEC_TEST_DIR).resolve() suite_path = pathlib.Path(SPEC_TEST_DIR).resolve()
if not suite_path.exists(): if not suite_path.exists():
@ -239,13 +256,19 @@ def test_suite(
xip_flag, xip_flag,
clean_up_flag, clean_up_flag,
verbose_flag, verbose_flag,
qemu_flag,
qemu_firmware,
], ],
) )
for case_name, result in results.items(): for case_name, result in results.items():
try: try:
# 5 min / case if qemu_flag:
result.wait(300) # 60 min / case, testing on QEMU may be very slow
result.wait(7200)
else:
# 5 min / case
result.wait(300)
if not result.successful(): if not result.successful():
failed_case += 1 failed_case += 1
else: else:
@ -268,6 +291,8 @@ def test_suite(
xip_flag, xip_flag,
clean_up_flag, clean_up_flag,
verbose_flag, verbose_flag,
qemu_flag,
qemu_firmware,
) )
successful_case += 1 successful_case += 1
except Exception as e: except Exception as e:
@ -349,6 +374,19 @@ def main():
dest="parl_flag", dest="parl_flag",
help="To run whole test suite parallelly", help="To run whole test suite parallelly",
) )
parser.add_argument(
"--qemu",
action="store_true",
default=False,
dest="qemu_flag",
help="To run whole test suite in qemu",
)
parser.add_argument(
"--qemu-firmware",
default="",
dest="qemu_firmware",
help="Firmware required by qemu",
)
parser.add_argument( parser.add_argument(
"--quiet", "--quiet",
action="store_false", action="store_false",
@ -389,6 +427,8 @@ def main():
options.clean_up_flag, options.clean_up_flag,
options.verbose_flag, options.verbose_flag,
options.parl_flag, options.parl_flag,
options.qemu_flag,
options.qemu_firmware,
) )
end = time.time_ns() end = time.time_ns()
print( print(
@ -408,6 +448,8 @@ def main():
options.xip_flag, options.xip_flag,
options.clean_up_flag, options.clean_up_flag,
options.verbose_flag, options.verbose_flag,
options.qemu_flag,
options.qemu_firmware
) )
else: else:
ret = True ret = True

266
tests/wamr-test-suites/spec-test-script/runtest.py
View File

@ -1,21 +1,26 @@
#!/usr/bin/env python #!/usr/bin/env python
from __future__ import print_function from __future__ import print_function
import os, sys, re
import argparse, time
import signal, atexit, tempfile, subprocess
from subprocess import Popen, STDOUT, PIPE import argparse
from select import select import array
import atexit
# Pseudo-TTY and terminal manipulation import fcntl
import pty, array, fcntl, termios
import shutil
import struct
import math import math
import os
# Pseudo-TTY and terminal manipulation
import pty
import re
import shutil
import struct
import subprocess
import sys
import tempfile
import termios
import time
import traceback import traceback
from select import select
from subprocess import PIPE, STDOUT, Popen
if sys.version_info[0] == 2: if sys.version_info[0] == 2:
IS_PY_3 = False IS_PY_3 = False
@ -23,7 +28,7 @@ else:
IS_PY_3 = True IS_PY_3 = True
test_aot = False test_aot = False
# "x86_64", "i386", "aarch64", "armv7" or "thumbv7" # "x86_64", "i386", "aarch64", "armv7", "thumbv7", "riscv32_ilp32", "riscv32_ilp32d", "riscv32_lp64", "riscv64_lp64d"
test_target = "x86_64" test_target = "x86_64"
debug_file = None debug_file = None
@ -32,10 +37,8 @@ log_file = None
# to save the register module with self-define name # to save the register module with self-define name
temp_file_repo = [] temp_file_repo = []
# get current work directory # to save the mapping of module files in /tmp by name
current_work_directory = os.getcwd() temp_module_table = {}
# set temporal file directory
temp_file_directory = os.path.join(current_work_directory,"tempfile")
def debug(data): def debug(data):
if debug_file: if debug_file:
@ -51,6 +54,7 @@ def log(data, end='\n'):
# TODO: do we need to support '\n' too # TODO: do we need to support '\n' too
import platform import platform
if platform.system().find("CYGWIN_NT") >= 0: if platform.system().find("CYGWIN_NT") >= 0:
# TODO: this is weird, is this really right on Cygwin? # TODO: this is weird, is this really right on Cygwin?
sep = "\n\r\n" sep = "\n\r\n"
@ -205,9 +209,9 @@ parser.add_argument('test_file', type=argparse.FileType('r'),
parser.add_argument('--aot', action='store_true', parser.add_argument('--aot', action='store_true',
help="Test with AOT") help="Test with AOT")
parser.add_argument('--aot-target', type=str, parser.add_argument('--target', type=str,
default="x86_64", default="x86_64",
help="Set aot target") help="Set running target")
parser.add_argument('--sgx', action='store_true', parser.add_argument('--sgx', action='store_true',
help="Test SGX") help="Test SGX")
@ -218,9 +222,17 @@ parser.add_argument('--simd', default=False, action='store_true',
parser.add_argument('--xip', default=False, action='store_true', parser.add_argument('--xip', default=False, action='store_true',
help="Enable XIP") help="Enable XIP")
parser.add_argument('--multi-module', default=False, action='store_true',
help="Enable Multi-thread")
parser.add_argument('--multi-thread', default=False, action='store_true', parser.add_argument('--multi-thread', default=False, action='store_true',
help="Enable Multi-thread") help="Enable Multi-thread")
parser.add_argument('--qemu', default=False, action='store_true',
help="Enable QEMU")
parser.add_argument('--qemu-firmware', default='', help="Firmware required by qemu")
parser.add_argument('--verbose', default=False, action='store_true', parser.add_argument('--verbose', default=False, action='store_true',
help='show more logs') help='show more logs')
@ -637,10 +649,10 @@ def is_result_match_expected(out, expected):
def test_assert(r, opts, mode, cmd, expected): def test_assert(r, opts, mode, cmd, expected):
log("Testing(%s) %s = %s" % (mode, cmd, expected)) log("Testing(%s) %s = %s" % (mode, cmd, expected))
out = invoke(r, opts, cmd) out = invoke(r, opts, cmd)
outs = [''] + out.split('\n')[1:] if '\n' in out or ' ' in out:
out = outs[-1] outs = [''] + out.split('\n')[1:]
out = outs[-1]
if mode=='trap': if mode=='trap':
o = re.sub('^Exception: ', '', out) o = re.sub('^Exception: ', '', out)
@ -756,9 +768,11 @@ def test_assert_return(r, opts, form):
returns = re.split("\)\s*\(", m.group(3)[1:-1]) returns = re.split("\)\s*\(", m.group(3)[1:-1])
# processed numbers in strings # processed numbers in strings
expected = [parse_assertion_value(v)[1] for v in returns] expected = [parse_assertion_value(v)[1] for v in returns]
test_assert(r, opts, "return", "%s %s" % (func, " ".join(args)), ",".join(expected)) expected = ",".join(expected)
test_assert(r, opts, "return", "%s %s" % (func, " ".join(args)), expected)
elif not m and n: elif not m and n:
module = os.path.join(temp_file_directory,n.group(1)) module = temp_module_table[n.group(1)].split(".wasm")[0]
# assume the cmd is (assert_return(invoke $ABC "func")). # assume the cmd is (assert_return(invoke $ABC "func")).
# run the ABC.wasm firstly # run the ABC.wasm firstly
if test_aot: if test_aot:
@ -769,9 +783,7 @@ def test_assert_return(r, opts, form):
_, exc, _ = sys.exc_info() _, exc, _ = sys.exc_info()
log("Run wamrc failed:\n got: '%s'" % r.buf) log("Run wamrc failed:\n got: '%s'" % r.buf)
sys.exit(1) sys.exit(1)
r = run_wasm_with_repl(module+".wasm", module+".aot", opts, r) r = run_wasm_with_repl(module+".wasm", module+".aot" if test_aot else module, opts, r)
else:
r = run_wasm_with_repl(module+".wasm", None, opts, r)
# Wait for the initial prompt # Wait for the initial prompt
try: try:
assert_prompt(r, ['webassembly> '], opts.start_timeout, False) assert_prompt(r, ['webassembly> '], opts.start_timeout, False)
@ -824,6 +836,8 @@ def test_assert_trap(r, opts, form):
elif not m and n: elif not m and n:
module = n.group(1) module = n.group(1)
module = tempfile.gettempdir() + "/" + module
# will trigger the module named in assert_return(invoke $ABC). # will trigger the module named in assert_return(invoke $ABC).
# run the ABC.wasm firstly # run the ABC.wasm firstly
if test_aot: if test_aot:
@ -834,9 +848,7 @@ def test_assert_trap(r, opts, form):
_, exc, _ = sys.exc_info() _, exc, _ = sys.exc_info()
log("Run wamrc failed:\n got: '%s'" % r.buf) log("Run wamrc failed:\n got: '%s'" % r.buf)
sys.exit(1) sys.exit(1)
r = run_wasm_with_repl(module+".wasm", module+".aot", opts, r) r = run_wasm_with_repl(module+".wasm", module+".aot" if test_aot else module, opts, r)
else:
r = run_wasm_with_repl(module+".wasm", None, opts, r)
# Wait for the initial prompt # Wait for the initial prompt
try: try:
assert_prompt(r, ['webassembly> '], opts.start_timeout, False) assert_prompt(r, ['webassembly> '], opts.start_timeout, False)
@ -936,11 +948,15 @@ def compile_wasm_to_aot(wasm_tempfile, aot_tempfile, runner, opts, r):
elif test_target == "armv7": elif test_target == "armv7":
cmd += ["--target=armv7", "--target-abi=gnueabihf"] cmd += ["--target=armv7", "--target-abi=gnueabihf"]
elif test_target == "thumbv7": elif test_target == "thumbv7":
cmd += ["--target=thumbv7", "--target-abi=gnueabihf", "--cpu=cortex-a15"] cmd += ["--target=thumbv7", "--target-abi=gnueabihf", "--cpu=cortex-a9"]
elif test_target == "riscv64_lp64d": elif test_target == "riscv32_ilp32":
cmd += ["--target=riscv64", "--target-abi=lp64d"] cmd += ["--target=riscv32", "--target-abi=ilp32"]
elif test_target == "riscv32_ilp32d":
cmd += ["--target=riscv32", "--target-abi=ilp32d"]
elif test_target == "riscv64_lp64": elif test_target == "riscv64_lp64":
cmd += ["--target=riscv64", "--target-abi=lp64"] cmd += ["--target=riscv64", "--target-abi=lp64"]
elif test_target == "riscv64_lp64d":
cmd += ["--target=riscv64", "--target-abi=lp64d"]
else: else:
pass pass
@ -974,59 +990,57 @@ def compile_wasm_to_aot(wasm_tempfile, aot_tempfile, runner, opts, r):
return r return r
def run_wasm_with_repl(wasm_tempfile, aot_tempfile, opts, r): def run_wasm_with_repl(wasm_tempfile, aot_tempfile, opts, r):
if not test_aot: tmpfile = aot_tempfile if test_aot else wasm_tempfile
log("Starting interpreter for module '%s'" % wasm_tempfile) log("Starting interpreter for module '%s'" % tmpfile)
if opts.verbose:
cmd = [opts.interpreter, "--heap-size=0", "-v=5", "--repl", wasm_tempfile] cmd_iwasm = [opts.interpreter, "--heap-size=0", "-v=5" if opts.verbose else "-v=0", "--repl", tmpfile]
else:
cmd = [opts.interpreter, "--heap-size=0", "--repl", wasm_tempfile] if opts.multi_module:
cmd_iwasm.insert(1, "--module-path=" + (tempfile.gettempdir() if not opts.qemu else "/tmp" ))
if opts.qemu:
if opts.qemu_firmware == '':
raise Exception("QEMU firmware missing")
if opts.target == "thumbv7":
cmd = ["qemu-system-arm", "-semihosting", "-M", "sabrelite", "-m", "1024", "-smp", "4", "-nographic", "-kernel", opts.qemu_firmware]
elif opts.target == "riscv32_ilp32":
cmd = ["qemu-system-riscv32", "-semihosting", "-M", "virt,aclint=on", "-cpu", "rv32", "-smp", "8", "-nographic", "-bios", "none", "-kernel", opts.qemu_firmware]
elif opts.target == "riscv64_lp64":
cmd = ["qemu-system-riscv64", "-semihosting", "-M", "virt,aclint=on", "-cpu", "rv64", "-smp", "8", "-nographic", "-bios", "none", "-kernel", opts.qemu_firmware]
else: else:
log("Starting aot for module '%s'" % aot_tempfile) cmd = cmd_iwasm
if opts.verbose:
cmd = [opts.interpreter, "--heap-size=0", "-v=5", "--repl", aot_tempfile]
else:
cmd = [opts.interpreter, "--heap-size=0", "--repl", aot_tempfile]
log("Running: %s" % " ".join(cmd)) log("Running: %s" % " ".join(cmd))
if (r != None): if (r != None):
r.cleanup() r.cleanup()
r = Runner(cmd, no_pty=opts.no_pty) r = Runner(cmd, no_pty=opts.no_pty)
if opts.qemu:
r.read_to_prompt(['nsh> '], 10)
r.writeline("mount -t hostfs -o fs={} /tmp".format(tempfile.gettempdir()))
r.read_to_prompt(['nsh> '], 10)
r.writeline(" ".join(cmd_iwasm))
return r return r
def create_tmpfiles(wast_name): def create_tmpfiles(wast_name):
tempfiles = [] tempfiles = []
# make tempfile directory
if not os.path.exists(temp_file_directory):
os.mkdir(temp_file_directory)
def makefile(name):
open(name, "w").close()
# create temporal file with particular name
temp_wast_file = os.path.join(temp_file_directory, ""+ wast_name + ".wast")
if not os.path.exists(temp_wast_file):
makefile(temp_wast_file)
tempfiles.append(temp_wast_file)
# now we define the same file name as wast for wasm & aot
wasm_file = wast_name +".wasm"
temp_wasm_file = os.path.join(temp_file_directory, wasm_file)
if not os.path.exists(temp_wasm_file):
makefile(temp_wasm_file)
tempfiles.append(temp_wasm_file)
(t1fd, wast_tempfile) = tempfile.mkstemp(suffix=".wast")
(t2fd, wasm_tempfile) = tempfile.mkstemp(suffix=".wasm")
tempfiles.append(wast_tempfile)
tempfiles.append(wasm_tempfile)
if test_aot: if test_aot:
aot_file = wast_name +".aot" (t3fd, aot_tempfile) = tempfile.mkstemp(suffix=".aot")
temp_aot_file =os.path.join(temp_file_directory, aot_file) tempfiles.append(aot_tempfile)
if not os.path.exists(temp_aot_file):
makefile(temp_aot_file)
tempfiles.append(temp_aot_file)
# add these temp file to temporal repo, will be deleted when finishing the test # add these temp file to temporal repo, will be deleted when finishing the test
temp_file_repo.extend(tempfiles) temp_file_repo.extend(tempfiles)
return tempfiles return tempfiles
def test_assert_with_exception(form, wast_tempfile, wasm_tempfile, aot_tempfile, opts, r): def test_assert_with_exception(form, wast_tempfile, wasm_tempfile, aot_tempfile, opts, r, loadable = True):
details_inside_ast = get_module_exp_from_assert(form) details_inside_ast = get_module_exp_from_assert(form)
log("module is ....'%s'"%details_inside_ast[0]) log("module is ....'%s'"%details_inside_ast[0])
log("exception is ....'%s'"%details_inside_ast[1]) log("exception is ....'%s'"%details_inside_ast[1])
@ -1052,29 +1066,31 @@ def test_assert_with_exception(form, wast_tempfile, wasm_tempfile, aot_tempfile,
log("Run wamrc failed:\n expected: '%s'\n got: '%s'" % \ log("Run wamrc failed:\n expected: '%s'\n got: '%s'" % \
(expected, r.buf)) (expected, r.buf))
sys.exit(1) sys.exit(1)
r = run_wasm_with_repl(wasm_tempfile, aot_tempfile, opts, r)
else:
r = run_wasm_with_repl(wasm_tempfile, None, opts, r)
# Wait for the initial prompt r = run_wasm_with_repl(wasm_tempfile, aot_tempfile if test_aot else None, opts, r)
try:
assert_prompt(r, ['webassembly> '], opts.start_timeout, True) # Some module couldn't load so will raise an error directly, so shell prompt won't show here
except:
_, exc, _ = sys.exc_info() if loadable:
if (r.buf.find(expected) >= 0): # Wait for the initial prompt
log("Out exception includes expected one, pass:") try:
log(" Expected: %s" %expected) assert_prompt(r, ['webassembly> '], opts.start_timeout, True)
log(" Got: %s" % r.buf) except:
else: _, exc, _ = sys.exc_info()
raise Exception("Failed:\n expected: '%s'\n got: '%s'" % \ if (r.buf.find(expected) >= 0):
(expected, r.buf)) log("Out exception includes expected one, pass:")
log(" Expected: %s" %expected)
log(" Got: %s" % r.buf)
else:
raise Exception("Failed:\n expected: '%s'\n got: '%s'" % \
(expected, r.buf))
if __name__ == "__main__": if __name__ == "__main__":
opts = parser.parse_args(sys.argv[1:]) opts = parser.parse_args(sys.argv[1:])
if opts.aot: test_aot = True if opts.aot: test_aot = True
# default x86_64 # default x86_64
test_target = opts.aot_target test_target = opts.target
if opts.rundir: os.chdir(opts.rundir) if opts.rundir: os.chdir(opts.rundir)
@ -1106,17 +1122,11 @@ if __name__ == "__main__":
elif skip_test(form, SKIP_TESTS): elif skip_test(form, SKIP_TESTS):
log("Skipping test: %s" % form[0:60]) log("Skipping test: %s" % form[0:60])
elif re.match("^\(assert_trap\s+\(module", form): elif re.match("^\(assert_trap\s+\(module", form):
if test_aot: test_assert_with_exception(form, wast_tempfile, wasm_tempfile, aot_tempfile if test_aot else None, opts, r)
test_assert_with_exception(form, wast_tempfile, wasm_tempfile, aot_tempfile, opts, r)
else:
test_assert_with_exception(form, wast_tempfile, wasm_tempfile, None, opts, r)
elif re.match("^\(assert_exhaustion\\b.*", form): elif re.match("^\(assert_exhaustion\\b.*", form):
test_assert_exhaustion(r, opts, form) test_assert_exhaustion(r, opts, form)
elif re.match("^\(assert_unlinkable\\b.*", form): elif re.match("^\(assert_unlinkable\\b.*", form):
if test_aot: test_assert_with_exception(form, wast_tempfile, wasm_tempfile, aot_tempfile if test_aot else None, opts, r, False)
test_assert_with_exception(form, wast_tempfile, wasm_tempfile, aot_tempfile, opts, r)
else:
test_assert_with_exception(form, wast_tempfile, wasm_tempfile, None, opts, r)
elif re.match("^\(assert_malformed\\b.*", form): elif re.match("^\(assert_malformed\\b.*", form):
# remove comments in wast # remove comments in wast
form,n = re.subn(";;.*\n", "", form) form,n = re.subn(";;.*\n", "", form)
@ -1153,35 +1163,22 @@ if __name__ == "__main__":
log("Run wamrc failed:\n expected: '%s'\n got: '%s'" % \ log("Run wamrc failed:\n expected: '%s'\n got: '%s'" % \
(error_msg, r.buf)) (error_msg, r.buf))
continue continue
cmd = [opts.interpreter, "--heap-size=0", "--repl", aot_tempfile]
else:
cmd = [opts.interpreter, "--heap-size=0", "--repl", wasm_tempfile]
log("Running: %s" % " ".join(cmd))
output = subprocess.check_output(cmd)
if IS_PY_3: r = run_wasm_with_repl(wasm_tempfile, aot_tempfile if test_aot else None, opts, r)
output = str(output, "latin1")
if (error_msg == "unexpected end of section or function") \ if (error_msg == "unexpected end of section or function"):
and output.endswith("unexpected end\n"):
# one case in binary.wast # one case in binary.wast
pass assert_prompt(r, ["unexpected end", error_msg], opts.start_timeout, True)
elif (error_msg == "invalid value type") \ elif (error_msg == "invalid value type"):
and output.endswith("unexpected end\n"):
# one case in binary.wast # one case in binary.wast
pass assert_prompt(r, ["unexpected end", error_msg], opts.start_timeout, True)
elif (error_msg == "length out of bounds") \ elif (error_msg == "length out of bounds"):
and output.endswith("unexpected end\n"):
# one case in custom.wast # one case in custom.wast
pass assert_prompt(r, ["unexpected end", error_msg], opts.start_timeout, True)
elif (error_msg == "integer representation too long") \ elif (error_msg == "integer representation too long"):
and output.endswith("invalid section id\n"):
# several cases in binary-leb128.wast # several cases in binary-leb128.wast
pass assert_prompt(r, ["invalid section id", error_msg], opts.start_timeout, True)
elif not error_msg in output:
raise Exception("Failed:\n expected: '%s'\n got: '%s'" % (error_msg, output[0:-1]))
else:
pass
elif re.match("^\(assert_malformed\s*\(module quote", form): elif re.match("^\(assert_malformed\s*\(module quote", form):
log("ignoring assert_malformed module quote") log("ignoring assert_malformed module quote")
else: else:
@ -1215,9 +1212,8 @@ if __name__ == "__main__":
_, exc, _ = sys.exc_info() _, exc, _ = sys.exc_info()
log("Run wamrc failed:\n got: '%s'" % r.buf) log("Run wamrc failed:\n got: '%s'" % r.buf)
sys.exit(1) sys.exit(1)
r = run_wasm_with_repl(temp_files[1], temp_files[2], opts, r) temp_module_table[module_name] = temp_files[1]
else: r = run_wasm_with_repl(temp_files[1], temp_files[2] if test_aot else None, opts, r)
r = run_wasm_with_repl(temp_files[1], None, opts, r)
else: else:
if not compile_wast_to_wasm(form, wast_tempfile, wasm_tempfile, opts): if not compile_wast_to_wasm(form, wast_tempfile, wasm_tempfile, opts):
raise Exception("compile wast to wasm failed") raise Exception("compile wast to wasm failed")
@ -1230,9 +1226,8 @@ if __name__ == "__main__":
_, exc, _ = sys.exc_info() _, exc, _ = sys.exc_info()
log("Run wamrc failed:\n got: '%s'" % r.buf) log("Run wamrc failed:\n got: '%s'" % r.buf)
sys.exit(1) sys.exit(1)
r = run_wasm_with_repl(wasm_tempfile, aot_tempfile, opts, r)
else: r = run_wasm_with_repl(wasm_tempfile, aot_tempfile if test_aot else None, opts, r)
r = run_wasm_with_repl(wasm_tempfile, None, opts, r)
# Wait for the initial prompt # Wait for the initial prompt
try: try:
@ -1251,32 +1246,14 @@ if __name__ == "__main__":
assert(r), "iwasm repl runtime should be not null" assert(r), "iwasm repl runtime should be not null"
do_invoke(r, opts, form) do_invoke(r, opts, form)
elif re.match("^\(assert_invalid\\b.*", form): elif re.match("^\(assert_invalid\\b.*", form):
if test_aot: test_assert_with_exception(form, wast_tempfile, wasm_tempfile, aot_tempfile if test_aot else None, opts, r)
test_assert_with_exception(form, wast_tempfile, wasm_tempfile, aot_tempfile, opts, r)
else:
test_assert_with_exception(form, wast_tempfile, wasm_tempfile, None, opts, r)
elif re.match("^\(register\\b.*", form): elif re.match("^\(register\\b.*", form):
# get module's new name from the register cmd # get module's new name from the register cmd
name_new =re.split('\"',re.search('\".*\"',form).group(0))[1] name_new =re.split('\"',re.search('\".*\"',form).group(0))[1]
if name_new: if name_new:
# if the register cmd include the new and old module name. new_module = os.path.join(tempfile.gettempdir(), name_new + ".wasm")
# like: (register "new" $old) shutil.copyfile(temp_module_table.get(name_new, wasm_tempfile), new_module)
# we will replace the old with new name.
name_old = re.search('\$.*\)',form)
if name_old:
old_ = re.split('\W', re.search('\$.*\)',form).group(0))[1]
old_module = os.path.join(temp_file_directory,old_+".wasm")
else:
# like: (register "new")
# this kind of register cmd will be behind of a noramal module
# these modules' name are default temporal file name
# we replace them with new name.
old_module = wasm_tempfile
new_module = os.path.join(current_work_directory,name_new+".wasm")
shutil.copyfile(old_module,new_module)
# add new_module copied from the old into temp_file_repo[] # add new_module copied from the old into temp_file_repo[]
temp_file_repo.append(new_module) temp_file_repo.append(new_module)
else: else:
@ -1303,9 +1280,6 @@ if __name__ == "__main__":
for t in temp_file_repo: for t in temp_file_repo:
if(len(str(t))!=0 and os.path.exists(t)): if(len(str(t))!=0 and os.path.exists(t)):
os.remove(t) os.remove(t)
# remove /tempfiles/ directory
if os.path.exists(temp_file_directory):
shutil.rmtree(temp_file_directory)
log("### End testing %s" % opts.test_file.name) log("### End testing %s" % opts.test_file.name)
else: else:

40
tests/wamr-test-suites/test_wamr.sh
View File

@ -24,6 +24,8 @@ function help()
echo "-x test SGX" echo "-x test SGX"
echo "-b use the wabt binary release package instead of compiling from the source code" echo "-b use the wabt binary release package instead of compiling from the source code"
echo "-P run the spec test parallelly" echo "-P run the spec test parallelly"
echo "-Q enable qemu"
echo "-F set the firmware path used by qemu"
} }
OPT_PARSED="" OPT_PARSED=""
@ -42,8 +44,10 @@ TEST_CASE_ARR=()
SGX_OPT="" SGX_OPT=""
PLATFORM=$(uname -s | tr A-Z a-z) PLATFORM=$(uname -s | tr A-Z a-z)
PARALLELISM=0 PARALLELISM=0
ENABLE_QEMU=0
QEMU_FIRMWARE=""
while getopts ":s:cabt:m:MCpSXxP" opt while getopts ":s:cabt:m:MCpSXxPQF:" opt
do do
OPT_PARSED="TRUE" OPT_PARSED="TRUE"
case $opt in case $opt in
@ -119,6 +123,14 @@ do
P) P)
PARALLELISM=1 PARALLELISM=1
;; ;;
Q)
echo "enable QEMU"
ENABLE_QEMU=1
;;
F)
echo "QEMU firmware" ${OPTARG}
QEMU_FIRMWARE=${OPTARG}
;;
?) ?)
help help
exit 1;; exit 1;;
@ -403,15 +415,23 @@ function spec_test()
ARGS_FOR_SPEC_TEST+="-X " ARGS_FOR_SPEC_TEST+="-X "
fi fi
# set the current running target
ARGS_FOR_SPEC_TEST+="-m ${TARGET} "
# require warmc only in aot mode # require warmc only in aot mode
if [[ $1 == 'aot' ]]; then if [[ $1 == 'aot' ]]; then
ARGS_FOR_SPEC_TEST+="-t -m ${TARGET} " ARGS_FOR_SPEC_TEST+="-t "
fi fi
if [[ ${PARALLELISM} == 1 ]]; then if [[ ${PARALLELISM} == 1 ]]; then
ARGS_FOR_SPEC_TEST+="--parl " ARGS_FOR_SPEC_TEST+="--parl "
fi fi
if [[ ${ENABLE_QEMU} == 1 ]]; then
ARGS_FOR_SPEC_TEST+="--qemu "
ARGS_FOR_SPEC_TEST+="--qemu-firmware ${QEMU_FIRMWARE}"
fi
cd ${WORK_DIR} cd ${WORK_DIR}
echo "python3 ./all.py ${ARGS_FOR_SPEC_TEST} | tee -a ${REPORT_DIR}/spec_test_report.txt" echo "python3 ./all.py ${ARGS_FOR_SPEC_TEST} | tee -a ${REPORT_DIR}/spec_test_report.txt"
python3 ./all.py ${ARGS_FOR_SPEC_TEST} | tee -a ${REPORT_DIR}/spec_test_report.txt python3 ./all.py ${ARGS_FOR_SPEC_TEST} | tee -a ${REPORT_DIR}/spec_test_report.txt
@ -509,8 +529,8 @@ function build_iwasm_with_cfg()
function build_wamrc() function build_wamrc()
{ {
if [[ $TARGET == "ARMV7_VFP" || $TARGET == "THUMBV7_VFP" if [[ $TARGET == "ARMV7_VFP" || $TARGET == "THUMBV7_VFP"
|| $TARGET == "RISCV64" || $TARGET == "RISCV64_LP64D" || $TARGET == "RISCV32" || $TARGET == "RISCV32_ILP32" || $TARGET == "RISCV32_ILP32D"
|| $TARGET == "RISCV64_LP64" ]];then || $TARGET == "RISCV64" || $TARGET == "RISCV64_LP64D" || $TARGET == "RISCV64_LP64" ]];then
echo "suppose wamrc is already built" echo "suppose wamrc is already built"
return return
fi fi
@ -581,7 +601,9 @@ function trigger()
echo "work in classic-interp mode" echo "work in classic-interp mode"
# classic-interp # classic-interp
BUILD_FLAGS="$CLASSIC_INTERP_COMPILE_FLAGS $EXTRA_COMPILE_FLAGS" BUILD_FLAGS="$CLASSIC_INTERP_COMPILE_FLAGS $EXTRA_COMPILE_FLAGS"
build_iwasm_with_cfg $BUILD_FLAGS if [[ ${ENABLE_QEMU} == 0 ]]; then
build_iwasm_with_cfg $BUILD_FLAGS
fi
for suite in "${TEST_CASE_ARR[@]}"; do for suite in "${TEST_CASE_ARR[@]}"; do
$suite"_test" classic-interp $suite"_test" classic-interp
done done
@ -597,7 +619,9 @@ function trigger()
echo "work in fast-interp mode" echo "work in fast-interp mode"
# fast-interp # fast-interp
BUILD_FLAGS="$FAST_INTERP_COMPILE_FLAGS $EXTRA_COMPILE_FLAGS" BUILD_FLAGS="$FAST_INTERP_COMPILE_FLAGS $EXTRA_COMPILE_FLAGS"
build_iwasm_with_cfg $BUILD_FLAGS if [[ ${ENABLE_QEMU} == 0 ]]; then
build_iwasm_with_cfg $BUILD_FLAGS
fi
for suite in "${TEST_CASE_ARR[@]}"; do for suite in "${TEST_CASE_ARR[@]}"; do
$suite"_test" fast-interp $suite"_test" fast-interp
done done
@ -631,7 +655,9 @@ function trigger()
echo "work in aot mode" echo "work in aot mode"
# aot # aot
BUILD_FLAGS="$AOT_COMPILE_FLAGS $EXTRA_COMPILE_FLAGS" BUILD_FLAGS="$AOT_COMPILE_FLAGS $EXTRA_COMPILE_FLAGS"
build_iwasm_with_cfg $BUILD_FLAGS if [[ ${ENABLE_QEMU} == 0 ]]; then
build_iwasm_with_cfg $BUILD_FLAGS
fi
build_wamrc build_wamrc
for suite in "${TEST_CASE_ARR[@]}"; do for suite in "${TEST_CASE_ARR[@]}"; do
$suite"_test" aot $suite"_test" aot