From 3f6ffe059f64b3ebc44dc0bbc63452cb8850702b Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 14 Sep 2018 00:44:06 +1000 Subject: [PATCH] py/objgenerator: Implement PEP479, StopIteration convs to RuntimeError. This commit implements PEP479 which disallows raising StopIteration inside a generator to signal that it should be finished. Instead, the generator should simply return when it is complete. See https://www.python.org/dev/peps/pep-0479/ for details. --- py/objgenerator.c | 18 +++++---------- tests/basics/gen_yield_from.py | 28 ----------------------- tests/basics/gen_yield_from.py.exp | 14 ------------ tests/basics/gen_yield_from_close.py | 6 ++--- tests/basics/gen_yield_from_close.py.exp | 20 ---------------- tests/basics/gen_yield_from_throw.py | 18 +++++++++++++-- tests/basics/gen_yield_from_throw.py.exp | 6 ----- tests/basics/generator_close.py | 5 ++-- tests/basics/generator_close.py.exp | 10 -------- tests/basics/generator_pep479.py | 29 ++++++++++++++++++++++++ tests/basics/generator_pep479.py.exp | 5 ++++ tests/run-tests | 2 +- 12 files changed, 63 insertions(+), 98 deletions(-) delete mode 100644 tests/basics/gen_yield_from.py.exp delete mode 100644 tests/basics/gen_yield_from_close.py.exp delete mode 100644 tests/basics/gen_yield_from_throw.py.exp delete mode 100644 tests/basics/generator_close.py.exp create mode 100644 tests/basics/generator_pep479.py create mode 100644 tests/basics/generator_pep479.py.exp diff --git a/py/objgenerator.c b/py/objgenerator.c index c45bebacd2..341967dc02 100644 --- a/py/objgenerator.c +++ b/py/objgenerator.c @@ -145,6 +145,10 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_ size_t n_state = mp_decode_uint_value(self->code_state.fun_bc->bytecode); self->code_state.ip = 0; *ret_val = self->code_state.state[n_state - 1]; + // PEP479: if StopIteration is raised inside a generator it is replaced with RuntimeError + if (mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(mp_obj_get_type(*ret_val)), MP_OBJ_FROM_PTR(&mp_type_StopIteration))) { + *ret_val = mp_obj_new_exception_msg(&mp_type_RuntimeError, "generator raised StopIteration"); + } break; } } @@ -168,15 +172,6 @@ STATIC mp_obj_t gen_resume_and_raise(mp_obj_t self_in, mp_obj_t send_value, mp_o return ret; case MP_VM_RETURN_EXCEPTION: - // TODO: Optimization of returning MP_OBJ_STOP_ITERATION is really part - // of mp_iternext() protocol, but this function is called by other methods - // too, which may not handled MP_OBJ_STOP_ITERATION. - if (mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(mp_obj_get_type(ret)), MP_OBJ_FROM_PTR(&mp_type_StopIteration))) { - mp_obj_t val = mp_obj_exception_get_value(ret); - if (val == mp_const_none) { - return MP_OBJ_STOP_ITERATION; - } - } nlr_raise(ret); } } @@ -216,11 +211,10 @@ STATIC mp_obj_t gen_instance_close(mp_obj_t self_in) { case MP_VM_RETURN_YIELD: mp_raise_msg(&mp_type_RuntimeError, "generator ignored GeneratorExit"); - // Swallow StopIteration & GeneratorExit (== successful close), and re-raise any other + // Swallow GeneratorExit (== successful close), and re-raise any other case MP_VM_RETURN_EXCEPTION: // ret should always be an instance of an exception class - if (mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(mp_obj_get_type(ret)), MP_OBJ_FROM_PTR(&mp_type_GeneratorExit)) || - mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(mp_obj_get_type(ret)), MP_OBJ_FROM_PTR(&mp_type_StopIteration))) { + if (mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(mp_obj_get_type(ret)), MP_OBJ_FROM_PTR(&mp_type_GeneratorExit))) { return mp_const_none; } nlr_raise(ret); diff --git a/tests/basics/gen_yield_from.py b/tests/basics/gen_yield_from.py index 4e68aec63b..037644e1ec 100644 --- a/tests/basics/gen_yield_from.py +++ b/tests/basics/gen_yield_from.py @@ -13,34 +13,6 @@ g = gen2() print(list(g)) -# Like above, but terminate subgen using StopIteration -def gen3(): - yield 1 - yield 2 - raise StopIteration - -def gen4(): - print("here1") - print((yield from gen3())) - print("here2") - -g = gen4() -print(list(g)) - -# Like above, but terminate subgen using StopIteration with value -def gen5(): - yield 1 - yield 2 - raise StopIteration(123) - -def gen6(): - print("here1") - print((yield from gen5())) - print("here2") - -g = gen6() -print(list(g)) - # StopIteration from within a Python function, within a native iterator (map), within a yield from def gen7(x): if x < 3: diff --git a/tests/basics/gen_yield_from.py.exp b/tests/basics/gen_yield_from.py.exp deleted file mode 100644 index 507f2b9caf..0000000000 --- a/tests/basics/gen_yield_from.py.exp +++ /dev/null @@ -1,14 +0,0 @@ -here1 -3 -here2 -[1, 2] -here1 -None -here2 -[1, 2] -here1 -123 -here2 -[1, 2] -444 -[0, 1, 2] diff --git a/tests/basics/gen_yield_from_close.py b/tests/basics/gen_yield_from_close.py index 8339861056..e3e0116ff7 100644 --- a/tests/basics/gen_yield_from_close.py +++ b/tests/basics/gen_yield_from_close.py @@ -55,14 +55,14 @@ except StopIteration: # Yet another variation - leaf generator gets GeneratorExit, -# but raises StopIteration instead. This still should close chain properly. +# and reraises a new GeneratorExit. This still should close chain properly. def gen5(): yield 1 try: yield 2 except GeneratorExit: - print("leaf caught GeneratorExit and raised StopIteration instead") - raise StopIteration(123) + print("leaf caught GeneratorExit and reraised GeneratorExit") + raise GeneratorExit(123) yield 3 yield 4 diff --git a/tests/basics/gen_yield_from_close.py.exp b/tests/basics/gen_yield_from_close.py.exp deleted file mode 100644 index a44d1353df..0000000000 --- a/tests/basics/gen_yield_from_close.py.exp +++ /dev/null @@ -1,20 +0,0 @@ --1 -1 -StopIteration --1 -1 -2 -leaf caught GeneratorExit and swallowed it -delegating caught GeneratorExit -StopIteration --1 -1 -2 -leaf caught GeneratorExit and raised StopIteration instead -delegating caught GeneratorExit -StopIteration -123 -RuntimeError -0 -1 -close diff --git a/tests/basics/gen_yield_from_throw.py b/tests/basics/gen_yield_from_throw.py index 829bf0f3b4..4d65b3c170 100644 --- a/tests/basics/gen_yield_from_throw.py +++ b/tests/basics/gen_yield_from_throw.py @@ -25,6 +25,20 @@ def gen3(): g3 = gen3() print(next(g3)) try: - g3.throw(StopIteration) + g3.throw(KeyError) +except KeyError: + print('got KeyError from downstream!') + +# case where a thrown exception is caught and stops the generator +def gen4(): + try: + yield 1 + yield 2 + except: + pass +g4 = gen4() +print(next(g4)) +try: + g4.throw(ValueError) except StopIteration: - print('got StopIteration from downstream!') + print('got StopIteration') diff --git a/tests/basics/gen_yield_from_throw.py.exp b/tests/basics/gen_yield_from_throw.py.exp deleted file mode 100644 index 6ce97ad86e..0000000000 --- a/tests/basics/gen_yield_from_throw.py.exp +++ /dev/null @@ -1,6 +0,0 @@ -1 -got ValueError from upstream! -str1 -got TypeError from downstream! -123 -got StopIteration from downstream! diff --git a/tests/basics/generator_close.py b/tests/basics/generator_close.py index aa563f2a8a..1ccc78dbe4 100644 --- a/tests/basics/generator_close.py +++ b/tests/basics/generator_close.py @@ -31,13 +31,14 @@ except StopIteration: print("StopIteration") -# Throwing StopIteration in response to close() is ok +# Throwing GeneratorExit in response to close() is ok def gen2(): try: yield 1 yield 2 except: - raise StopIteration + print('raising GeneratorExit') + raise GeneratorExit g = gen2() next(g) diff --git a/tests/basics/generator_close.py.exp b/tests/basics/generator_close.py.exp deleted file mode 100644 index fcd5839357..0000000000 --- a/tests/basics/generator_close.py.exp +++ /dev/null @@ -1,10 +0,0 @@ -None -StopIteration -1 -None -StopIteration -[1, 2] -None -StopIteration -None -ValueError diff --git a/tests/basics/generator_pep479.py b/tests/basics/generator_pep479.py new file mode 100644 index 0000000000..e422c349e7 --- /dev/null +++ b/tests/basics/generator_pep479.py @@ -0,0 +1,29 @@ +# tests for correct PEP479 behaviour (introduced in Python 3.5) + +# basic case: StopIteration is converted into a RuntimeError +def gen(): + yield 1 + raise StopIteration +g = gen() +print(next(g)) +try: + next(g) +except RuntimeError: + print('RuntimeError') + +# trying to continue a failed generator now raises StopIteration +try: + next(g) +except StopIteration: + print('StopIteration') + +# throwing a StopIteration which is uncaught will be converted into a RuntimeError +def gen(): + yield 1 + yield 2 +g = gen() +print(next(g)) +try: + g.throw(StopIteration) +except RuntimeError: + print('RuntimeError') diff --git a/tests/basics/generator_pep479.py.exp b/tests/basics/generator_pep479.py.exp new file mode 100644 index 0000000000..c64fe49561 --- /dev/null +++ b/tests/basics/generator_pep479.py.exp @@ -0,0 +1,5 @@ +1 +RuntimeError +StopIteration +1 +RuntimeError diff --git a/tests/run-tests b/tests/run-tests index 8c087f9f58..4da9ccaece 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -352,7 +352,7 @@ def run_tests(pyb, tests, args, base_path="."): # Some tests are known to fail with native emitter # Remove them from the below when they work if args.emit == 'native': - skip_tests.update({'basics/%s.py' % t for t in 'gen_yield_from gen_yield_from_close gen_yield_from_ducktype gen_yield_from_exc gen_yield_from_executing gen_yield_from_iter gen_yield_from_send gen_yield_from_stopped gen_yield_from_throw gen_yield_from_throw2 gen_yield_from_throw3 generator1 generator2 generator_args generator_close generator_closure generator_exc generator_name generator_pend_throw generator_return generator_send'.split()}) # require yield + skip_tests.update({'basics/%s.py' % t for t in 'gen_yield_from gen_yield_from_close gen_yield_from_ducktype gen_yield_from_exc gen_yield_from_executing gen_yield_from_iter gen_yield_from_send gen_yield_from_stopped gen_yield_from_throw gen_yield_from_throw2 gen_yield_from_throw3 generator1 generator2 generator_args generator_close generator_closure generator_exc generator_name generator_pend_throw generator_return generator_send generator_pep479'.split()}) # require yield skip_tests.update({'basics/%s.py' % t for t in 'bytes_gen class_store_class globals_del string_join'.split()}) # require yield skip_tests.update({'basics/async_%s.py' % t for t in 'def await await2 for for2 with with2 with_break with_return'.split()}) # require yield skip_tests.update({'basics/%s.py' % t for t in 'try_reraise try_reraise2'.split()}) # require raise_varargs