mirror
/
micropython
kopia lustrzana https://github.com/micropython/micropython
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

tests/run-tests.py: Reformat with Black. 

Signed-off-by: Damien George <damien@micropython.org>
pull/7014/head
Damien George 2021-03-11 16:11:27 +11:00
rodzic 6129b8e401
commit 2a38d71036
1 zmienionych plików z 351 dodań i 229 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

580
tests/run-tests.py
Wyświetl plik

@ -13,31 +13,34 @@ from glob import glob
# are guaranteed to always work, this one should though. # are guaranteed to always work, this one should though.
BASEPATH = os.path.dirname(os.path.abspath(inspect.getsourcefile(lambda: None))) BASEPATH = os.path.dirname(os.path.abspath(inspect.getsourcefile(lambda: None)))
def base_path(*p): def base_path(*p):
return os.path.abspath(os.path.join(BASEPATH, *p)).replace('\\', '/') return os.path.abspath(os.path.join(BASEPATH, *p)).replace("\\", "/")
# Tests require at least CPython 3.3. If your default python3 executable # Tests require at least CPython 3.3. If your default python3 executable
# is of lower version, you can point MICROPY_CPYTHON3 environment var # is of lower version, you can point MICROPY_CPYTHON3 environment var
# to the correct executable. # to the correct executable.
if os.name == 'nt': if os.name == "nt":
CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python') CPYTHON3 = os.getenv("MICROPY_CPYTHON3", "python")
MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', base_path('../ports/windows/micropython.exe')) MICROPYTHON = os.getenv("MICROPY_MICROPYTHON", base_path("../ports/windows/micropython.exe"))
else: else:
CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3') CPYTHON3 = os.getenv("MICROPY_CPYTHON3", "python3")
MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', base_path('../ports/unix/micropython')) MICROPYTHON = os.getenv("MICROPY_MICROPYTHON", base_path("../ports/unix/micropython"))
# Use CPython options to not save .pyc files, to only access the core standard library # Use CPython options to not save .pyc files, to only access the core standard library
# (not site packages which may clash with u-module names), and improve start up time. # (not site packages which may clash with u-module names), and improve start up time.
CPYTHON3_CMD = [CPYTHON3, "-BS"] CPYTHON3_CMD = [CPYTHON3, "-BS"]
# mpy-cross is only needed if --via-mpy command-line arg is passed # mpy-cross is only needed if --via-mpy command-line arg is passed
MPYCROSS = os.getenv('MICROPY_MPYCROSS', base_path('../mpy-cross/mpy-cross')) MPYCROSS = os.getenv("MICROPY_MPYCROSS", base_path("../mpy-cross/mpy-cross"))
# For diff'ing test output # For diff'ing test output
DIFF = os.getenv('MICROPY_DIFF', 'diff -u') DIFF = os.getenv("MICROPY_DIFF", "diff -u")
# Set PYTHONIOENCODING so that CPython will use utf-8 on systems which set another encoding in the locale # Set PYTHONIOENCODING so that CPython will use utf-8 on systems which set another encoding in the locale
os.environ['PYTHONIOENCODING'] = 'utf-8' os.environ["PYTHONIOENCODING"] = "utf-8"
def rm_f(fname): def rm_f(fname):
if os.path.exists(fname): if os.path.exists(fname):
@ -48,57 +51,62 @@ def rm_f(fname):
def convert_regex_escapes(line): def convert_regex_escapes(line):
cs = [] cs = []
escape = False escape = False
for c in str(line, 'utf8'): for c in str(line, "utf8"):
if escape: if escape:
escape = False escape = False
cs.append(c) cs.append(c)
elif c == '\\': elif c == "\\":
escape = True escape = True
elif c in ('(', ')', '[', ']', '{', '}', '.', '*', '+', '^', '$'): elif c in ("(", ")", "[", "]", "{", "}", ".", "*", "+", "^", "$"):
cs.append('\\' + c) cs.append("\\" + c)
else: else:
cs.append(c) cs.append(c)
# accept carriage-return(s) before final newline # accept carriage-return(s) before final newline
if cs[-1] == '\n': if cs[-1] == "\n":
cs[-1] = '\r*\n' cs[-1] = "\r*\n"
return bytes(''.join(cs), 'utf8') return bytes("".join(cs), "utf8")
def run_micropython(pyb, args, test_file, is_special=False): def run_micropython(pyb, args, test_file, is_special=False):
special_tests = ( special_tests = (
'micropython/meminfo.py', 'basics/bytes_compare3.py', "micropython/meminfo.py",
'basics/builtin_help.py', 'thread/thread_exc2.py', "basics/bytes_compare3.py",
'esp32/partition_ota.py', "basics/builtin_help.py",
"thread/thread_exc2.py",
"esp32/partition_ota.py",
) )
had_crash = False had_crash = False
if pyb is None: if pyb is None:
# run on PC # run on PC
if test_file.startswith(('cmdline/', base_path('feature_check/'))) or test_file in special_tests: if (
test_file.startswith(("cmdline/", base_path("feature_check/")))
or test_file in special_tests
):
# special handling for tests of the unix cmdline program # special handling for tests of the unix cmdline program
is_special = True is_special = True
if is_special: if is_special:
# check for any cmdline options needed for this test # check for any cmdline options needed for this test
args = [MICROPYTHON] args = [MICROPYTHON]
with open(test_file, 'rb') as f: with open(test_file, "rb") as f:
line = f.readline() line = f.readline()
if line.startswith(b'# cmdline:'): if line.startswith(b"# cmdline:"):
# subprocess.check_output on Windows only accepts strings, not bytes # subprocess.check_output on Windows only accepts strings, not bytes
args += [str(c, 'utf-8') for c in line[10:].strip().split()] args += [str(c, "utf-8") for c in line[10:].strip().split()]
# run the test, possibly with redirected input # run the test, possibly with redirected input
try: try:
if 'repl_' in test_file: if "repl_" in test_file:
# Need to use a PTY to test command line editing # Need to use a PTY to test command line editing
try: try:
import pty import pty
except ImportError: except ImportError:
# in case pty module is not available, like on Windows # in case pty module is not available, like on Windows
return b'SKIP\n' return b"SKIP\n"
import select import select
def get(required=False): def get(required=False):
rv = b'' rv = b""
while True: while True:
ready = select.select([master], [], [], 0.02) ready = select.select([master], [], [], 0.02)
if ready[0] == [master]: if ready[0] == [master]:
@ -111,14 +119,15 @@ def run_micropython(pyb, args, test_file, is_special=False):
os.write(master, what) os.write(master, what)
return get() return get()
with open(test_file, 'rb') as f: with open(test_file, "rb") as f:
# instead of: output_mupy = subprocess.check_output(args, stdin=f) # instead of: output_mupy = subprocess.check_output(args, stdin=f)
master, slave = pty.openpty() master, slave = pty.openpty()
p = subprocess.Popen(args, stdin=slave, stdout=slave, p = subprocess.Popen(
stderr=subprocess.STDOUT, bufsize=0) args, stdin=slave, stdout=slave, stderr=subprocess.STDOUT, bufsize=0
)
banner = get(True) banner = get(True)
output_mupy = banner + b''.join(send_get(line) for line in f) output_mupy = banner + b"".join(send_get(line) for line in f)
send_get(b'\x04') # exit the REPL, so coverage info is saved send_get(b"\x04") # exit the REPL, so coverage info is saved
# At this point the process might have exited already, but trying to # At this point the process might have exited already, but trying to
# kill it 'again' normally doesn't result in exceptions as Python and/or # kill it 'again' normally doesn't result in exceptions as Python and/or
# the OS seem to try to handle this nicely. When running Linux on WSL # the OS seem to try to handle this nicely. When running Linux on WSL
@ -132,22 +141,28 @@ def run_micropython(pyb, args, test_file, is_special=False):
os.close(master) os.close(master)
os.close(slave) os.close(slave)
else: else:
output_mupy = subprocess.check_output(args + [test_file], stderr=subprocess.STDOUT) output_mupy = subprocess.check_output(
args + [test_file], stderr=subprocess.STDOUT
)
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
return b'CRASH' return b"CRASH"
else: else:
# a standard test run on PC # a standard test run on PC
# create system command # create system command
cmdlist = [MICROPYTHON, '-X', 'emit=' + args.emit] cmdlist = [MICROPYTHON, "-X", "emit=" + args.emit]
if args.heapsize is not None: if args.heapsize is not None:
cmdlist.extend(['-X', 'heapsize=' + args.heapsize]) cmdlist.extend(["-X", "heapsize=" + args.heapsize])
# if running via .mpy, first compile the .py file # if running via .mpy, first compile the .py file
if args.via_mpy: if args.via_mpy:
subprocess.check_output([MPYCROSS] + args.mpy_cross_flags.split() + ['-o', 'mpytest.mpy', '-X', 'emit=' + args.emit, test_file]) subprocess.check_output(
cmdlist.extend(['-m', 'mpytest']) [MPYCROSS]
+ args.mpy_cross_flags.split()
+ ["-o", "mpytest.mpy", "-X", "emit=" + args.emit, test_file]
)
cmdlist.extend(["-m", "mpytest"])
else: else:
cmdlist.append(test_file) cmdlist.append(test_file)
@ -156,11 +171,11 @@ def run_micropython(pyb, args, test_file, is_special=False):
output_mupy = subprocess.check_output(cmdlist, stderr=subprocess.STDOUT) output_mupy = subprocess.check_output(cmdlist, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as er: except subprocess.CalledProcessError as er:
had_crash = True had_crash = True
output_mupy = er.output + b'CRASH' output_mupy = er.output + b"CRASH"
# clean up if we had an intermediate .mpy file # clean up if we had an intermediate .mpy file
if args.via_mpy: if args.via_mpy:
rm_f('mpytest.mpy') rm_f("mpytest.mpy")
else: else:
# run on pyboard # run on pyboard
@ -169,56 +184,58 @@ def run_micropython(pyb, args, test_file, is_special=False):
output_mupy = pyb.execfile(test_file) output_mupy = pyb.execfile(test_file)
except pyboard.PyboardError as e: except pyboard.PyboardError as e:
had_crash = True had_crash = True
if not is_special and e.args[0] == 'exception': if not is_special and e.args[0] == "exception":
output_mupy = e.args[1] + e.args[2] + b'CRASH' output_mupy = e.args[1] + e.args[2] + b"CRASH"
else: else:
output_mupy = b'CRASH' output_mupy = b"CRASH"
# canonical form for all ports/platforms is to use \n for end-of-line # canonical form for all ports/platforms is to use \n for end-of-line
output_mupy = output_mupy.replace(b'\r\n', b'\n') output_mupy = output_mupy.replace(b"\r\n", b"\n")
# don't try to convert the output if we should skip this test # don't try to convert the output if we should skip this test
if had_crash or output_mupy in (b'SKIP\n', b'CRASH'): if had_crash or output_mupy in (b"SKIP\n", b"CRASH"):
return output_mupy return output_mupy
if is_special or test_file in special_tests: if is_special or test_file in special_tests:
# convert parts of the output that are not stable across runs # convert parts of the output that are not stable across runs
with open(test_file + '.exp', 'rb') as f: with open(test_file + ".exp", "rb") as f:
lines_exp = [] lines_exp = []
for line in f.readlines(): for line in f.readlines():
if line == b'########\n': if line == b"########\n":
line = (line,) line = (line,)
else: else:
line = (line, re.compile(convert_regex_escapes(line))) line = (line, re.compile(convert_regex_escapes(line)))
lines_exp.append(line) lines_exp.append(line)
lines_mupy = [line + b'\n' for line in output_mupy.split(b'\n')] lines_mupy = [line + b"\n" for line in output_mupy.split(b"\n")]
if output_mupy.endswith(b'\n'): if output_mupy.endswith(b"\n"):
lines_mupy = lines_mupy[:-1] # remove erroneous last empty line lines_mupy = lines_mupy[:-1] # remove erroneous last empty line
i_mupy = 0 i_mupy = 0
for i in range(len(lines_exp)): for i in range(len(lines_exp)):
if lines_exp[i][0] == b'########\n': if lines_exp[i][0] == b"########\n":
# 8x #'s means match 0 or more whole lines # 8x #'s means match 0 or more whole lines
line_exp = lines_exp[i + 1] line_exp = lines_exp[i + 1]
skip = 0 skip = 0
while i_mupy + skip < len(lines_mupy) and not line_exp[1].match(lines_mupy[i_mupy + skip]): while i_mupy + skip < len(lines_mupy) and not line_exp[1].match(
lines_mupy[i_mupy + skip]
):
skip += 1 skip += 1
if i_mupy + skip >= len(lines_mupy): if i_mupy + skip >= len(lines_mupy):
lines_mupy[i_mupy] = b'######## FAIL\n' lines_mupy[i_mupy] = b"######## FAIL\n"
break break
del lines_mupy[i_mupy:i_mupy + skip] del lines_mupy[i_mupy : i_mupy + skip]
lines_mupy.insert(i_mupy, b'########\n') lines_mupy.insert(i_mupy, b"########\n")
i_mupy += 1 i_mupy += 1
else: else:
# a regex # a regex
if lines_exp[i][1].match(lines_mupy[i_mupy]): if lines_exp[i][1].match(lines_mupy[i_mupy]):
lines_mupy[i_mupy] = lines_exp[i][0] lines_mupy[i_mupy] = lines_exp[i][0]
else: else:
#print("don't match: %r %s" % (lines_exp[i][1], lines_mupy[i_mupy])) # DEBUG # print("don't match: %r %s" % (lines_exp[i][1], lines_mupy[i_mupy])) # DEBUG
pass pass
i_mupy += 1 i_mupy += 1
if i_mupy >= len(lines_mupy): if i_mupy >= len(lines_mupy):
break break
output_mupy = b''.join(lines_mupy) output_mupy = b"".join(lines_mupy)
return output_mupy return output_mupy
@ -261,193 +278,220 @@ def run_tests(pyb, tests, args, result_dir):
# run-tests.py script itself so use base_path. # run-tests.py script itself so use base_path.
# Check if micropython.native is supported, and skip such tests if it's not # Check if micropython.native is supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, 'native_check.py') output = run_feature_check(pyb, args, base_path, "native_check.py")
if output == b'CRASH': if output == b"CRASH":
skip_native = True skip_native = True
# Check if arbitrary-precision integers are supported, and skip such tests if it's not # Check if arbitrary-precision integers are supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, 'int_big.py') output = run_feature_check(pyb, args, base_path, "int_big.py")
if output != b'1000000000000000000000000000000000000000000000\n': if output != b"1000000000000000000000000000000000000000000000\n":
skip_int_big = True skip_int_big = True
# Check if bytearray is supported, and skip such tests if it's not # Check if bytearray is supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, 'bytearray.py') output = run_feature_check(pyb, args, base_path, "bytearray.py")
if output != b'bytearray\n': if output != b"bytearray\n":
skip_bytearray = True skip_bytearray = True
# Check if set type (and set literals) is supported, and skip such tests if it's not # Check if set type (and set literals) is supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, 'set_check.py') output = run_feature_check(pyb, args, base_path, "set_check.py")
if output == b'CRASH': if output == b"CRASH":
skip_set_type = True skip_set_type = True
# Check if slice is supported, and skip such tests if it's not # Check if slice is supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, 'slice.py') output = run_feature_check(pyb, args, base_path, "slice.py")
if output != b'slice\n': if output != b"slice\n":
skip_slice = True skip_slice = True
# Check if async/await keywords are supported, and skip such tests if it's not # Check if async/await keywords are supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, 'async_check.py') output = run_feature_check(pyb, args, base_path, "async_check.py")
if output == b'CRASH': if output == b"CRASH":
skip_async = True skip_async = True
# Check if const keyword (MicroPython extension) is supported, and skip such tests if it's not # Check if const keyword (MicroPython extension) is supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, 'const.py') output = run_feature_check(pyb, args, base_path, "const.py")
if output == b'CRASH': if output == b"CRASH":
skip_const = True skip_const = True
# Check if __rOP__ special methods are supported, and skip such tests if it's not # Check if __rOP__ special methods are supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, 'reverse_ops.py') output = run_feature_check(pyb, args, base_path, "reverse_ops.py")
if output == b'TypeError\n': if output == b"TypeError\n":
skip_revops = True skip_revops = True
# Check if uio module exists, and skip such tests if it doesn't # Check if uio module exists, and skip such tests if it doesn't
output = run_feature_check(pyb, args, base_path, 'uio_module.py') output = run_feature_check(pyb, args, base_path, "uio_module.py")
if output != b'uio\n': if output != b"uio\n":
skip_io_module = True skip_io_module = True
# Check if emacs repl is supported, and skip such tests if it's not # Check if emacs repl is supported, and skip such tests if it's not
t = run_feature_check(pyb, args, base_path, 'repl_emacs_check.py') t = run_feature_check(pyb, args, base_path, "repl_emacs_check.py")
if 'True' not in str(t, 'ascii'): if "True" not in str(t, "ascii"):
skip_tests.add('cmdline/repl_emacs_keys.py') skip_tests.add("cmdline/repl_emacs_keys.py")
# Check if words movement in repl is supported, and skip such tests if it's not # Check if words movement in repl is supported, and skip such tests if it's not
t = run_feature_check(pyb, args, base_path, 'repl_words_move_check.py') t = run_feature_check(pyb, args, base_path, "repl_words_move_check.py")
if 'True' not in str(t, 'ascii'): if "True" not in str(t, "ascii"):
skip_tests.add('cmdline/repl_words_move.py') skip_tests.add("cmdline/repl_words_move.py")
upy_byteorder = run_feature_check(pyb, args, base_path, 'byteorder.py') upy_byteorder = run_feature_check(pyb, args, base_path, "byteorder.py")
upy_float_precision = run_feature_check(pyb, args, base_path, 'float.py') upy_float_precision = run_feature_check(pyb, args, base_path, "float.py")
if upy_float_precision == b'CRASH': if upy_float_precision == b"CRASH":
upy_float_precision = 0 upy_float_precision = 0
else: else:
upy_float_precision = int(upy_float_precision) upy_float_precision = int(upy_float_precision)
has_complex = run_feature_check(pyb, args, base_path, 'complex.py') == b'complex\n' has_complex = run_feature_check(pyb, args, base_path, "complex.py") == b"complex\n"
has_coverage = run_feature_check(pyb, args, base_path, 'coverage.py') == b'coverage\n' has_coverage = run_feature_check(pyb, args, base_path, "coverage.py") == b"coverage\n"
cpy_byteorder = subprocess.check_output(CPYTHON3_CMD + [base_path('feature_check/byteorder.py')]) cpy_byteorder = subprocess.check_output(
skip_endian = (upy_byteorder != cpy_byteorder) CPYTHON3_CMD + [base_path("feature_check/byteorder.py")]
)
skip_endian = upy_byteorder != cpy_byteorder
# These tests don't test slice explicitly but rather use it to perform the test # These tests don't test slice explicitly but rather use it to perform the test
misc_slice_tests = ( misc_slice_tests = (
'builtin_range', "builtin_range",
'class_super', "class_super",
'containment', "containment",
'errno1', "errno1",
'fun_str', "fun_str",
'generator1', "generator1",
'globals_del', "globals_del",
'memoryview1', "memoryview1",
'memoryview_gc', "memoryview_gc",
'object1', "object1",
'python34', "python34",
'struct_endian', "struct_endian",
) )
# Some tests shouldn't be run on GitHub Actions # Some tests shouldn't be run on GitHub Actions
if os.getenv('GITHUB_ACTIONS') == 'true': if os.getenv("GITHUB_ACTIONS") == "true":
skip_tests.add('thread/stress_schedule.py') # has reliability issues skip_tests.add("thread/stress_schedule.py") # has reliability issues
if upy_float_precision == 0: if upy_float_precision == 0:
skip_tests.add('extmod/uctypes_le_float.py') skip_tests.add("extmod/uctypes_le_float.py")
skip_tests.add('extmod/uctypes_native_float.py') skip_tests.add("extmod/uctypes_native_float.py")
skip_tests.add('extmod/uctypes_sizeof_float.py') skip_tests.add("extmod/uctypes_sizeof_float.py")
skip_tests.add('extmod/ujson_dumps_float.py') skip_tests.add("extmod/ujson_dumps_float.py")
skip_tests.add('extmod/ujson_loads_float.py') skip_tests.add("extmod/ujson_loads_float.py")
skip_tests.add('extmod/urandom_extra_float.py') skip_tests.add("extmod/urandom_extra_float.py")
skip_tests.add('misc/rge_sm.py') skip_tests.add("misc/rge_sm.py")
if upy_float_precision < 32: if upy_float_precision < 32:
skip_tests.add('float/float2int_intbig.py') # requires fp32, there's float2int_fp30_intbig.py instead skip_tests.add(
skip_tests.add('float/string_format.py') # requires fp32, there's string_format_fp30.py instead "float/float2int_intbig.py"
skip_tests.add('float/bytes_construct.py') # requires fp32 ) # requires fp32, there's float2int_fp30_intbig.py instead
skip_tests.add('float/bytearray_construct.py') # requires fp32 skip_tests.add(
"float/string_format.py"
) # requires fp32, there's string_format_fp30.py instead
skip_tests.add("float/bytes_construct.py") # requires fp32
skip_tests.add("float/bytearray_construct.py") # requires fp32
if upy_float_precision < 64: if upy_float_precision < 64:
skip_tests.add('float/float_divmod.py') # tested by float/float_divmod_relaxed.py instead skip_tests.add("float/float_divmod.py") # tested by float/float_divmod_relaxed.py instead
skip_tests.add('float/float2int_doubleprec_intbig.py') skip_tests.add("float/float2int_doubleprec_intbig.py")
skip_tests.add('float/float_parse_doubleprec.py') skip_tests.add("float/float_parse_doubleprec.py")
if not has_complex: if not has_complex:
skip_tests.add('float/complex1.py') skip_tests.add("float/complex1.py")
skip_tests.add('float/complex1_intbig.py') skip_tests.add("float/complex1_intbig.py")
skip_tests.add('float/complex_special_methods.py') skip_tests.add("float/complex_special_methods.py")
skip_tests.add('float/int_big_float.py') skip_tests.add("float/int_big_float.py")
skip_tests.add('float/true_value.py') skip_tests.add("float/true_value.py")
skip_tests.add('float/types.py') skip_tests.add("float/types.py")
if not has_coverage: if not has_coverage:
skip_tests.add('cmdline/cmd_parsetree.py') skip_tests.add("cmdline/cmd_parsetree.py")
# Some tests shouldn't be run on a PC # Some tests shouldn't be run on a PC
if args.target == 'unix': if args.target == "unix":
# unix build does not have the GIL so can't run thread mutation tests # unix build does not have the GIL so can't run thread mutation tests
for t in tests: for t in tests:
if t.startswith('thread/mutate_'): if t.startswith("thread/mutate_"):
skip_tests.add(t) skip_tests.add(t)
# Some tests shouldn't be run on pyboard # Some tests shouldn't be run on pyboard
if args.target != 'unix': if args.target != "unix":
skip_tests.add('basics/exception_chain.py') # warning is not printed skip_tests.add("basics/exception_chain.py") # warning is not printed
skip_tests.add('micropython/meminfo.py') # output is very different to PC output skip_tests.add("micropython/meminfo.py") # output is very different to PC output
skip_tests.add('extmod/machine_mem.py') # raw memory access not supported skip_tests.add("extmod/machine_mem.py") # raw memory access not supported
if args.target == 'wipy': if args.target == "wipy":
skip_tests.add('misc/print_exception.py') # requires error reporting full skip_tests.add("misc/print_exception.py") # requires error reporting full
skip_tests.update({'extmod/uctypes_%s.py' % t for t in 'bytearray le native_le ptr_le ptr_native_le sizeof sizeof_native array_assign_le array_assign_native_le'.split()}) # requires uctypes skip_tests.update(
skip_tests.add('extmod/zlibd_decompress.py') # requires zlib {
skip_tests.add('extmod/uheapq1.py') # uheapq not supported by WiPy "extmod/uctypes_%s.py" % t
skip_tests.add('extmod/urandom_basic.py') # requires urandom for t in "bytearray le native_le ptr_le ptr_native_le sizeof sizeof_native array_assign_le array_assign_native_le".split()
skip_tests.add('extmod/urandom_extra.py') # requires urandom }
elif args.target == 'esp8266': ) # requires uctypes
skip_tests.add('misc/rge_sm.py') # too large skip_tests.add("extmod/zlibd_decompress.py") # requires zlib
elif args.target == 'minimal': skip_tests.add("extmod/uheapq1.py") # uheapq not supported by WiPy
skip_tests.add('basics/class_inplace_op.py') # all special methods not supported skip_tests.add("extmod/urandom_basic.py") # requires urandom
skip_tests.add('basics/subclass_native_init.py')# native subclassing corner cases not support skip_tests.add("extmod/urandom_extra.py") # requires urandom
skip_tests.add('misc/rge_sm.py') # too large elif args.target == "esp8266":
skip_tests.add('micropython/opt_level.py') # don't assume line numbers are stored skip_tests.add("misc/rge_sm.py") # too large
elif args.target == 'nrf': elif args.target == "minimal":
skip_tests.add('basics/memoryview1.py') # no item assignment for memoryview skip_tests.add("basics/class_inplace_op.py") # all special methods not supported
skip_tests.add('extmod/urandom_basic.py') # unimplemented: urandom.seed skip_tests.add(
skip_tests.add('micropython/opt_level.py') # no support for line numbers "basics/subclass_native_init.py"
skip_tests.add('misc/non_compliant.py') # no item assignment for bytearray ) # native subclassing corner cases not support
skip_tests.add("misc/rge_sm.py") # too large
skip_tests.add("micropython/opt_level.py") # don't assume line numbers are stored
elif args.target == "nrf":
skip_tests.add("basics/memoryview1.py") # no item assignment for memoryview
skip_tests.add("extmod/urandom_basic.py") # unimplemented: urandom.seed
skip_tests.add("micropython/opt_level.py") # no support for line numbers
skip_tests.add("misc/non_compliant.py") # no item assignment for bytearray
for t in tests: for t in tests:
if t.startswith('basics/io_'): if t.startswith("basics/io_"):
skip_tests.add(t) skip_tests.add(t)
elif args.target == 'qemu-arm': elif args.target == "qemu-arm":
skip_tests.add('misc/print_exception.py') # requires sys stdfiles skip_tests.add("misc/print_exception.py") # requires sys stdfiles
# Some tests are known to fail on 64-bit machines # Some tests are known to fail on 64-bit machines
if pyb is None and platform.architecture()[0] == '64bit': if pyb is None and platform.architecture()[0] == "64bit":
pass pass
# Some tests use unsupported features on Windows # Some tests use unsupported features on Windows
if os.name == 'nt': if os.name == "nt":
skip_tests.add('import/import_file.py') # works but CPython prints forward slashes skip_tests.add("import/import_file.py") # works but CPython prints forward slashes
# Some tests are known to fail with native emitter # Some tests are known to fail with native emitter
# Remove them from the below when they work # Remove them from the below when they work
if args.emit == 'native': if args.emit == "native":
skip_tests.update({'basics/%s.py' % t for t in 'gen_yield_from_close generator_name'.split()}) # require raise_varargs, generator name skip_tests.update(
skip_tests.update({'basics/async_%s.py' % t for t in 'with with2 with_break with_return'.split()}) # require async_with {"basics/%s.py" % t for t in "gen_yield_from_close generator_name".split()}
skip_tests.update({'basics/%s.py' % t for t in 'try_reraise try_reraise2'.split()}) # require raise_varargs ) # require raise_varargs, generator name
skip_tests.add('basics/annotate_var.py') # requires checking for unbound local skip_tests.update(
skip_tests.add('basics/del_deref.py') # requires checking for unbound local {"basics/async_%s.py" % t for t in "with with2 with_break with_return".split()}
skip_tests.add('basics/del_local.py') # requires checking for unbound local ) # require async_with
skip_tests.add('basics/exception_chain.py') # raise from is not supported skip_tests.update(
skip_tests.add('basics/scope_implicit.py') # requires checking for unbound local {"basics/%s.py" % t for t in "try_reraise try_reraise2".split()}
skip_tests.add('basics/try_finally_return2.py') # requires raise_varargs ) # require raise_varargs
skip_tests.add('basics/unboundlocal.py') # requires checking for unbound local skip_tests.add("basics/annotate_var.py") # requires checking for unbound local
skip_tests.add('extmod/uasyncio_event.py') # unknown issue skip_tests.add("basics/del_deref.py") # requires checking for unbound local
skip_tests.add('extmod/uasyncio_lock.py') # requires async with skip_tests.add("basics/del_local.py") # requires checking for unbound local
skip_tests.add('extmod/uasyncio_micropython.py') # unknown issue skip_tests.add("basics/exception_chain.py") # raise from is not supported
skip_tests.add('extmod/uasyncio_wait_for.py') # unknown issue skip_tests.add("basics/scope_implicit.py") # requires checking for unbound local
skip_tests.add('misc/features.py') # requires raise_varargs skip_tests.add("basics/try_finally_return2.py") # requires raise_varargs
skip_tests.add('misc/print_exception.py') # because native doesn't have proper traceback info skip_tests.add("basics/unboundlocal.py") # requires checking for unbound local
skip_tests.add('misc/sys_exc_info.py') # sys.exc_info() is not supported for native skip_tests.add("extmod/uasyncio_event.py") # unknown issue
skip_tests.add('micropython/emg_exc.py') # because native doesn't have proper traceback info skip_tests.add("extmod/uasyncio_lock.py") # requires async with
skip_tests.add('micropython/heapalloc_traceback.py') # because native doesn't have proper traceback info skip_tests.add("extmod/uasyncio_micropython.py") # unknown issue
skip_tests.add('micropython/opt_level_lineno.py') # native doesn't have proper traceback info skip_tests.add("extmod/uasyncio_wait_for.py") # unknown issue
skip_tests.add('micropython/schedule.py') # native code doesn't check pending events skip_tests.add("misc/features.py") # requires raise_varargs
skip_tests.add(
"misc/print_exception.py"
) # because native doesn't have proper traceback info
skip_tests.add("misc/sys_exc_info.py") # sys.exc_info() is not supported for native
skip_tests.add(
"micropython/emg_exc.py"
) # because native doesn't have proper traceback info
skip_tests.add(
"micropython/heapalloc_traceback.py"
) # because native doesn't have proper traceback info
skip_tests.add(
"micropython/opt_level_lineno.py"
) # native doesn't have proper traceback info
skip_tests.add("micropython/schedule.py") # native code doesn't check pending events
for test_file in tests: for test_file in tests:
test_file = test_file.replace('\\', '/') test_file = test_file.replace("\\", "/")
if args.filters: if args.filters:
# Default verdict is the opposit of the first action # Default verdict is the opposit of the first action
@ -458,9 +502,13 @@ def run_tests(pyb, tests, args, result_dir):
if verdict == "exclude": if verdict == "exclude":
continue continue
test_basename = test_file.replace('..', '_').replace('./', '').replace('/', '_') test_basename = test_file.replace("..", "_").replace("./", "").replace("/", "_")
test_name = os.path.splitext(os.path.basename(test_file))[0] test_name = os.path.splitext(os.path.basename(test_file))[0]
is_native = test_name.startswith("native_") or test_name.startswith("viper_") or args.emit == "native" is_native = (
test_name.startswith("native_")
or test_name.startswith("viper_")
or args.emit == "native"
)
is_endian = test_name.endswith("_endian") is_endian = test_name.endswith("_endian")
is_int_big = test_name.startswith("int_big") or test_name.endswith("_intbig") is_int_big = test_name.startswith("int_big") or test_name.endswith("_intbig")
is_bytearray = test_name.startswith("bytearray") or test_name.endswith("_bytearray") is_bytearray = test_name.startswith("bytearray") or test_name.endswith("_bytearray")
@ -493,23 +541,23 @@ def run_tests(pyb, tests, args, result_dir):
continue continue
# get expected output # get expected output
test_file_expected = test_file + '.exp' test_file_expected = test_file + ".exp"
if os.path.isfile(test_file_expected): if os.path.isfile(test_file_expected):
# expected output given by a file, so read that in # expected output given by a file, so read that in
with open(test_file_expected, 'rb') as f: with open(test_file_expected, "rb") as f:
output_expected = f.read() output_expected = f.read()
else: else:
# run CPython to work out expected output # run CPython to work out expected output
try: try:
output_expected = subprocess.check_output(CPYTHON3_CMD + [test_file]) output_expected = subprocess.check_output(CPYTHON3_CMD + [test_file])
if args.write_exp: if args.write_exp:
with open(test_file_expected, 'wb') as f: with open(test_file_expected, "wb") as f:
f.write(output_expected) f.write(output_expected)
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
output_expected = b'CPYTHON3 CRASH' output_expected = b"CPYTHON3 CRASH"
# canonical form for all host platforms is to use \n for end-of-line # canonical form for all host platforms is to use \n for end-of-line
output_expected = output_expected.replace(b'\r\n', b'\n') output_expected = output_expected.replace(b"\r\n", b"\n")
if args.write_exp: if args.write_exp:
continue continue
@ -517,7 +565,7 @@ def run_tests(pyb, tests, args, result_dir):
# run MicroPython # run MicroPython
output_mupy = run_micropython(pyb, args, test_file) output_mupy = run_micropython(pyb, args, test_file)
if output_mupy == b'SKIP\n': if output_mupy == b"SKIP\n":
print("skip ", test_file) print("skip ", test_file)
skipped_tests.append(test_name) skipped_tests.append(test_name)
continue continue
@ -549,9 +597,9 @@ def run_tests(pyb, tests, args, result_dir):
print("{} tests passed".format(passed_count)) print("{} tests passed".format(passed_count))
if len(skipped_tests) > 0: if len(skipped_tests) > 0:
print("{} tests skipped: {}".format(len(skipped_tests), ' '.join(skipped_tests))) print("{} tests skipped: {}".format(len(skipped_tests), " ".join(skipped_tests)))
if len(failed_tests) > 0: if len(failed_tests) > 0:
print("{} tests failed: {}".format(len(failed_tests), ' '.join(failed_tests))) print("{} tests failed: {}".format(len(failed_tests), " ".join(failed_tests)))
return False return False
# all tests succeeded # all tests succeeded
@ -559,7 +607,6 @@ def run_tests(pyb, tests, args, result_dir):
class append_filter(argparse.Action): class append_filter(argparse.Action):
def __init__(self, option_strings, dest, **kwargs): def __init__(self, option_strings, dest, **kwargs):
super().__init__(option_strings, dest, default=[], **kwargs) super().__init__(option_strings, dest, default=[], **kwargs)
@ -576,7 +623,7 @@ class append_filter(argparse.Action):
def main(): def main():
cmd_parser = argparse.ArgumentParser( cmd_parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter, formatter_class=argparse.RawDescriptionHelpFormatter,
description='''Run and manage tests for MicroPython. description="""Run and manage tests for MicroPython.
Tests are discovered by scanning test directories for .py files or using the Tests are discovered by scanning test directories for .py files or using the
specified test files. If test files nor directories are specified, the script specified test files. If test files nor directories are specified, the script
@ -587,34 +634,81 @@ produced by running the test through CPython unless a <test>.exp file is found,
case it is used as comparison. case it is used as comparison.
If a test fails, run-tests.py produces a pair of <test>.out and <test>.exp files in the result If a test fails, run-tests.py produces a pair of <test>.out and <test>.exp files in the result
directory with the MicroPython output and the expectations, respectively. directory with the MicroPython output and the expectations, respectively.
''', """,
epilog='''\ epilog="""\
Options -i and -e can be multiple and processed in the order given. Regex Options -i and -e can be multiple and processed in the order given. Regex
"search" (vs "match") operation is used. An action (include/exclude) of "search" (vs "match") operation is used. An action (include/exclude) of
the last matching regex is used: the last matching regex is used:
run-tests.py -i async - exclude all, then include tests containing "async" anywhere run-tests.py -i async - exclude all, then include tests containing "async" anywhere
run-tests.py -e '/big.+int' - include all, then exclude by regex run-tests.py -e '/big.+int' - include all, then exclude by regex
run-tests.py -e async -i async_foo - include all, exclude async, yet still include async_foo run-tests.py -e async -i async_foo - include all, exclude async, yet still include async_foo
''') """,
cmd_parser.add_argument('--target', default='unix', help='the target platform') )
cmd_parser.add_argument('--device', default='/dev/ttyACM0', help='the serial device or the IP address of the pyboard') cmd_parser.add_argument("--target", default="unix", help="the target platform")
cmd_parser.add_argument('-b', '--baudrate', default=115200, help='the baud rate of the serial device') cmd_parser.add_argument(
cmd_parser.add_argument('-u', '--user', default='micro', help='the telnet login username') "--device",
cmd_parser.add_argument('-p', '--password', default='python', help='the telnet login password') default="/dev/ttyACM0",
cmd_parser.add_argument('-d', '--test-dirs', nargs='*', help='input test directories (if no files given)') help="the serial device or the IP address of the pyboard",
cmd_parser.add_argument('-r', '--result-dir', default=base_path('results'), help='directory for test results') )
cmd_parser.add_argument('-e', '--exclude', action=append_filter, metavar='REGEX', dest='filters', help='exclude test by regex on path/name.py') cmd_parser.add_argument(
cmd_parser.add_argument('-i', '--include', action=append_filter, metavar='REGEX', dest='filters', help='include test by regex on path/name.py') "-b", "--baudrate", default=115200, help="the baud rate of the serial device"
cmd_parser.add_argument('--write-exp', action='store_true', help='use CPython to generate .exp files to run tests w/o CPython') )
cmd_parser.add_argument('--list-tests', action='store_true', help='list tests instead of running them') cmd_parser.add_argument("-u", "--user", default="micro", help="the telnet login username")
cmd_parser.add_argument('--emit', default='bytecode', help='MicroPython emitter to use (bytecode or native)') cmd_parser.add_argument("-p", "--password", default="python", help="the telnet login password")
cmd_parser.add_argument('--heapsize', help='heapsize to use (use default if not specified)') cmd_parser.add_argument(
cmd_parser.add_argument('--via-mpy', action='store_true', help='compile .py files to .mpy first') "-d", "--test-dirs", nargs="*", help="input test directories (if no files given)"
cmd_parser.add_argument('--mpy-cross-flags', default='-mcache-lookup-bc', help='flags to pass to mpy-cross') )
cmd_parser.add_argument('--keep-path', action='store_true', help='do not clear MICROPYPATH when running tests') cmd_parser.add_argument(
cmd_parser.add_argument('files', nargs='*', help='input test files') "-r", "--result-dir", default=base_path("results"), help="directory for test results"
cmd_parser.add_argument('--print-failures', action='store_true', help='print the diff of expected vs. actual output for failed tests and exit') )
cmd_parser.add_argument('--clean-failures', action='store_true', help='delete the .exp and .out files from failed tests and exit') cmd_parser.add_argument(
"-e",
"--exclude",
action=append_filter,
metavar="REGEX",
dest="filters",
help="exclude test by regex on path/name.py",
)
cmd_parser.add_argument(
"-i",
"--include",
action=append_filter,
metavar="REGEX",
dest="filters",
help="include test by regex on path/name.py",
)
cmd_parser.add_argument(
"--write-exp",
action="store_true",
help="use CPython to generate .exp files to run tests w/o CPython",
)
cmd_parser.add_argument(
"--list-tests", action="store_true", help="list tests instead of running them"
)
cmd_parser.add_argument(
"--emit", default="bytecode", help="MicroPython emitter to use (bytecode or native)"
)
cmd_parser.add_argument("--heapsize", help="heapsize to use (use default if not specified)")
cmd_parser.add_argument(
"--via-mpy", action="store_true", help="compile .py files to .mpy first"
)
cmd_parser.add_argument(
"--mpy-cross-flags", default="-mcache-lookup-bc", help="flags to pass to mpy-cross"
)
cmd_parser.add_argument(
"--keep-path", action="store_true", help="do not clear MICROPYPATH when running tests"
)
cmd_parser.add_argument("files", nargs="*", help="input test files")
cmd_parser.add_argument(
"--print-failures",
action="store_true",
help="print the diff of expected vs. actual output for failed tests and exit",
)
cmd_parser.add_argument(
"--clean-failures",
action="store_true",
help="delete the .exp and .out files from failed tests and exit",
)
args = cmd_parser.parse_args() args = cmd_parser.parse_args()
if args.print_failures: if args.print_failures:
@ -627,55 +721,82 @@ the last matching regex is used:
sys.exit(0) sys.exit(0)
if args.clean_failures: if args.clean_failures:
for f in glob(os.path.join(args.result_dir, "*.exp")) + glob(os.path.join(args.result_dir, "*.out")): for f in glob(os.path.join(args.result_dir, "*.exp")) + glob(
os.path.join(args.result_dir, "*.out")
):
os.remove(f) os.remove(f)
sys.exit(0) sys.exit(0)
LOCAL_TARGETS = ('unix', 'qemu-arm',) LOCAL_TARGETS = (
EXTERNAL_TARGETS = ('pyboard', 'wipy', 'esp8266', 'esp32', 'minimal', 'nrf') "unix",
"qemu-arm",
)
EXTERNAL_TARGETS = ("pyboard", "wipy", "esp8266", "esp32", "minimal", "nrf")
if args.target in LOCAL_TARGETS or args.list_tests: if args.target in LOCAL_TARGETS or args.list_tests:
pyb = None pyb = None
elif args.target in EXTERNAL_TARGETS: elif args.target in EXTERNAL_TARGETS:
global pyboard global pyboard
sys.path.append(base_path('../tools')) sys.path.append(base_path("../tools"))
import pyboard import pyboard
pyb = pyboard.Pyboard(args.device, args.baudrate, args.user, args.password) pyb = pyboard.Pyboard(args.device, args.baudrate, args.user, args.password)
pyb.enter_raw_repl() pyb.enter_raw_repl()
else: else:
raise ValueError('target must be one of %s' % ", ".join(LOCAL_TARGETS + EXTERNAL_TARGETS)) raise ValueError("target must be one of %s" % ", ".join(LOCAL_TARGETS + EXTERNAL_TARGETS))
if len(args.files) == 0: if len(args.files) == 0:
if args.test_dirs is None: if args.test_dirs is None:
test_dirs = ('basics', 'micropython', 'misc', 'extmod',) test_dirs = (
if args.target == 'pyboard': "basics",
"micropython",
"misc",
"extmod",
)
if args.target == "pyboard":
# run pyboard tests # run pyboard tests
test_dirs += ('float', 'stress', 'pyb', 'pybnative', 'inlineasm') test_dirs += ("float", "stress", "pyb", "pybnative", "inlineasm")
elif args.target in ('esp8266', 'esp32', 'minimal', 'nrf'): elif args.target in ("esp8266", "esp32", "minimal", "nrf"):
test_dirs += ('float',) test_dirs += ("float",)
elif args.target == 'wipy': elif args.target == "wipy":
# run WiPy tests # run WiPy tests
test_dirs += ('wipy',) test_dirs += ("wipy",)
elif args.target == 'unix': elif args.target == "unix":
# run PC tests # run PC tests
test_dirs += ('float', 'import', 'io', 'stress', 'unicode', 'unix', 'cmdline',) test_dirs += (
elif args.target == 'qemu-arm': "float",
"import",
"io",
"stress",
"unicode",
"unix",
"cmdline",
)
elif args.target == "qemu-arm":
if not args.write_exp: if not args.write_exp:
raise ValueError('--target=qemu-arm must be used with --write-exp') raise ValueError("--target=qemu-arm must be used with --write-exp")
# Generate expected output files for qemu run. # Generate expected output files for qemu run.
# This list should match the test_dirs tuple in tinytest-codegen.py. # This list should match the test_dirs tuple in tinytest-codegen.py.
test_dirs += ('float', 'inlineasm', 'qemu-arm',) test_dirs += (
"float",
"inlineasm",
"qemu-arm",
)
else: else:
# run tests from these directories # run tests from these directories
test_dirs = args.test_dirs test_dirs = args.test_dirs
tests = sorted(test_file for test_files in (glob('{}/*.py'.format(dir)) for dir in test_dirs) for test_file in test_files) tests = sorted(
test_file
for test_files in (glob("{}/*.py".format(dir)) for dir in test_dirs)
for test_file in test_files
)
else: else:
# tests explicitly given # tests explicitly given
tests = args.files tests = args.files
if not args.keep_path: if not args.keep_path:
# clear search path to make sure tests use only builtin modules and those in extmod # clear search path to make sure tests use only builtin modules and those in extmod
os.environ['MICROPYPATH'] = os.pathsep + base_path('../extmod') os.environ["MICROPYPATH"] = os.pathsep + base_path("../extmod")
try: try:
os.makedirs(args.result_dir, exist_ok=True) os.makedirs(args.result_dir, exist_ok=True)
@ -687,5 +808,6 @@ the last matching regex is used:
if not res: if not res:
sys.exit(1) sys.exit(1)
if __name__ == "__main__": if __name__ == "__main__":
main() main()